Total coverage: 256325 (17%)of 1557944 













































  983 












  229 







  158 







  236 
    1 

















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_KSM_H
#define __LINUX_KSM_H
/*
 * Memory merging support.
 *
 * This code enables dynamic sharing of identical pages found in different
 * memory areas, even if they are not shared by fork().
 */

#include <linux/bitops.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/rmap.h>
#include <linux/sched.h>
#include <linux/sched/coredump.h>

#ifdef CONFIG_KSM
int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
                unsigned long end, int advice, unsigned long *vm_flags);

void ksm_add_vma(struct vm_area_struct *vma);
int ksm_enable_merge_any(struct mm_struct *mm);
int ksm_disable_merge_any(struct mm_struct *mm);
int ksm_disable(struct mm_struct *mm);

int __ksm_enter(struct mm_struct *mm);
void __ksm_exit(struct mm_struct *mm);
/*
 * To identify zeropages that were mapped by KSM, we reuse the dirty bit
 * in the PTE. If the PTE is dirty, the zeropage was mapped by KSM when
 * deduplicating memory.
 */
#define is_ksm_zero_pte(pte)        (is_zero_pfn(pte_pfn(pte)) && pte_dirty(pte))

extern atomic_long_t ksm_zero_pages;

static inline void ksm_map_zero_page(struct mm_struct *mm)
{
        atomic_long_inc(&ksm_zero_pages);
        atomic_long_inc(&mm->ksm_zero_pages);
}

static inline void ksm_might_unmap_zero_page(struct mm_struct *mm, pte_t pte)
{
        if (is_ksm_zero_pte(pte)) {
                atomic_long_dec(&ksm_zero_pages);
                atomic_long_dec(&mm->ksm_zero_pages);
        }
}

static inline long mm_ksm_zero_pages(struct mm_struct *mm)
{
        return atomic_long_read(&mm->ksm_zero_pages);
}

static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
{
        if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags))
                return __ksm_enter(mm);

        return 0;
}

static inline int ksm_execve(struct mm_struct *mm)
{
        if (test_bit(MMF_VM_MERGE_ANY, &mm->flags))
                return __ksm_enter(mm);

        return 0;
}

static inline void ksm_exit(struct mm_struct *mm)
{
        if (test_bit(MMF_VM_MERGEABLE, &mm->flags))
                __ksm_exit(mm);
}

/*
 * When do_swap_page() first faults in from swap what used to be a KSM page,
 * no problem, it will be assigned to this vma's anon_vma; but thereafter,
 * it might be faulted into a different anon_vma (or perhaps to a different
 * offset in the same anon_vma).  do_swap_page() cannot do all the locking
 * needed to reconstitute a cross-anon_vma KSM page: for now it has to make
 * a copy, and leave remerging the pages to a later pass of ksmd.
 *
 * We'd like to make this conditional on vma->vm_flags & VM_MERGEABLE,
 * but what if the vma was unmerged while the page was swapped out?
 */
struct folio *ksm_might_need_to_copy(struct folio *folio,
                        struct vm_area_struct *vma, unsigned long addr);

void rmap_walk_ksm(struct folio *folio, struct rmap_walk_control *rwc);
void folio_migrate_ksm(struct folio *newfolio, struct folio *folio);
void collect_procs_ksm(struct folio *folio, struct page *page,
                struct list_head *to_kill, int force_early);
long ksm_process_profit(struct mm_struct *);

#else  /* !CONFIG_KSM */

static inline void ksm_add_vma(struct vm_area_struct *vma)
{
}

static inline int ksm_disable(struct mm_struct *mm)
{
        return 0;
}

static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
{
        return 0;
}

static inline int ksm_execve(struct mm_struct *mm)
{
        return 0;
}

static inline void ksm_exit(struct mm_struct *mm)
{
}

static inline void ksm_might_unmap_zero_page(struct mm_struct *mm, pte_t pte)
{
}

static inline void collect_procs_ksm(struct folio *folio, struct page *page,
                                     struct list_head *to_kill, int force_early)
{
}

#ifdef CONFIG_MMU
static inline int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
                unsigned long end, int advice, unsigned long *vm_flags)
{
        return 0;
}

static inline struct folio *ksm_might_need_to_copy(struct folio *folio,
                        struct vm_area_struct *vma, unsigned long addr)
{
        return folio;
}

static inline void rmap_walk_ksm(struct folio *folio,
                        struct rmap_walk_control *rwc)
{
}

static inline void folio_migrate_ksm(struct folio *newfolio, struct folio *old)
{
}
#endif /* CONFIG_MMU */
#endif /* !CONFIG_KSM */

#endif /* __LINUX_KSM_H */














































































   35 






   10 













    1 






   10 






    1 




































   42 










































   32 






   11 









































   60 






   61 














    2 






   61 






 3459 






   61 






 3455 










1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM cgroup

#if !defined(_TRACE_CGROUP_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_CGROUP_H

#include <linux/cgroup.h>
#include <linux/tracepoint.h>

DECLARE_EVENT_CLASS(cgroup_root,

        TP_PROTO(struct cgroup_root *root),

        TP_ARGS(root),

        TP_STRUCT__entry(
                __field(        int,                root                        )
                __field(        u16,                ss_mask                        )
                __string(        name,                root->name                )
        ),

        TP_fast_assign(
                __entry->root = root->hierarchy_id;
                __entry->ss_mask = root->subsys_mask;
                __assign_str(name);
        ),

        TP_printk("root=%d ss_mask=%#x name=%s",
                  __entry->root, __entry->ss_mask, __get_str(name))
);

DEFINE_EVENT(cgroup_root, cgroup_setup_root,

        TP_PROTO(struct cgroup_root *root),

        TP_ARGS(root)
);

DEFINE_EVENT(cgroup_root, cgroup_destroy_root,

        TP_PROTO(struct cgroup_root *root),

        TP_ARGS(root)
);

DEFINE_EVENT(cgroup_root, cgroup_remount,

        TP_PROTO(struct cgroup_root *root),

        TP_ARGS(root)
);

DECLARE_EVENT_CLASS(cgroup,

        TP_PROTO(struct cgroup *cgrp, const char *path),

        TP_ARGS(cgrp, path),

        TP_STRUCT__entry(
                __field(        int,                root                        )
                __field(        int,                level                        )
                __field(        u64,                id                        )
                __string(        path,                path                        )
        ),

        TP_fast_assign(
                __entry->root = cgrp->root->hierarchy_id;
                __entry->id = cgroup_id(cgrp);
                __entry->level = cgrp->level;
                __assign_str(path);
        ),

        TP_printk("root=%d id=%llu level=%d path=%s",
                  __entry->root, __entry->id, __entry->level, __get_str(path))
);

DEFINE_EVENT(cgroup, cgroup_mkdir,

        TP_PROTO(struct cgroup *cgrp, const char *path),

        TP_ARGS(cgrp, path)
);

DEFINE_EVENT(cgroup, cgroup_rmdir,

        TP_PROTO(struct cgroup *cgrp, const char *path),

        TP_ARGS(cgrp, path)
);

DEFINE_EVENT(cgroup, cgroup_release,

        TP_PROTO(struct cgroup *cgrp, const char *path),

        TP_ARGS(cgrp, path)
);

DEFINE_EVENT(cgroup, cgroup_rename,

        TP_PROTO(struct cgroup *cgrp, const char *path),

        TP_ARGS(cgrp, path)
);

DEFINE_EVENT(cgroup, cgroup_freeze,

        TP_PROTO(struct cgroup *cgrp, const char *path),

        TP_ARGS(cgrp, path)
);

DEFINE_EVENT(cgroup, cgroup_unfreeze,

        TP_PROTO(struct cgroup *cgrp, const char *path),

        TP_ARGS(cgrp, path)
);

DECLARE_EVENT_CLASS(cgroup_migrate,

        TP_PROTO(struct cgroup *dst_cgrp, const char *path,
                 struct task_struct *task, bool threadgroup),

        TP_ARGS(dst_cgrp, path, task, threadgroup),

        TP_STRUCT__entry(
                __field(        int,                dst_root                )
                __field(        int,                dst_level                )
                __field(        u64,                dst_id                        )
                __field(        int,                pid                        )
                __string(        dst_path,        path                        )
                __string(        comm,                task->comm                )
        ),

        TP_fast_assign(
                __entry->dst_root = dst_cgrp->root->hierarchy_id;
                __entry->dst_id = cgroup_id(dst_cgrp);
                __entry->dst_level = dst_cgrp->level;
                __assign_str(dst_path);
                __entry->pid = task->pid;
                __assign_str(comm);
        ),

        TP_printk("dst_root=%d dst_id=%llu dst_level=%d dst_path=%s pid=%d comm=%s",
                  __entry->dst_root, __entry->dst_id, __entry->dst_level,
                  __get_str(dst_path), __entry->pid, __get_str(comm))
);

DEFINE_EVENT(cgroup_migrate, cgroup_attach_task,

        TP_PROTO(struct cgroup *dst_cgrp, const char *path,
                 struct task_struct *task, bool threadgroup),

        TP_ARGS(dst_cgrp, path, task, threadgroup)
);

DEFINE_EVENT(cgroup_migrate, cgroup_transfer_tasks,

        TP_PROTO(struct cgroup *dst_cgrp, const char *path,
                 struct task_struct *task, bool threadgroup),

        TP_ARGS(dst_cgrp, path, task, threadgroup)
);

DECLARE_EVENT_CLASS(cgroup_event,

        TP_PROTO(struct cgroup *cgrp, const char *path, int val),

        TP_ARGS(cgrp, path, val),

        TP_STRUCT__entry(
                __field(        int,                root                        )
                __field(        int,                level                        )
                __field(        u64,                id                        )
                __string(        path,                path                        )
                __field(        int,                val                        )
        ),

        TP_fast_assign(
                __entry->root = cgrp->root->hierarchy_id;
                __entry->id = cgroup_id(cgrp);
                __entry->level = cgrp->level;
                __assign_str(path);
                __entry->val = val;
        ),

        TP_printk("root=%d id=%llu level=%d path=%s val=%d",
                  __entry->root, __entry->id, __entry->level, __get_str(path),
                  __entry->val)
);

DEFINE_EVENT(cgroup_event, cgroup_notify_populated,

        TP_PROTO(struct cgroup *cgrp, const char *path, int val),

        TP_ARGS(cgrp, path, val)
);

DEFINE_EVENT(cgroup_event, cgroup_notify_frozen,

        TP_PROTO(struct cgroup *cgrp, const char *path, int val),

        TP_ARGS(cgrp, path, val)
);

DECLARE_EVENT_CLASS(cgroup_rstat,

        TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),

        TP_ARGS(cgrp, cpu, contended),

        TP_STRUCT__entry(
                __field(        int,                root                        )
                __field(        int,                level                        )
                __field(        u64,                id                        )
                __field(        int,                cpu                        )
                __field(        bool,                contended                )
        ),

        TP_fast_assign(
                __entry->root = cgrp->root->hierarchy_id;
                __entry->id = cgroup_id(cgrp);
                __entry->level = cgrp->level;
                __entry->cpu = cpu;
                __entry->contended = contended;
        ),

        TP_printk("root=%d id=%llu level=%d cpu=%d lock contended:%d",
                  __entry->root, __entry->id, __entry->level,
                  __entry->cpu, __entry->contended)
);

/* Related to global: cgroup_rstat_lock */
DEFINE_EVENT(cgroup_rstat, cgroup_rstat_lock_contended,

        TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),

        TP_ARGS(cgrp, cpu, contended)
);

DEFINE_EVENT(cgroup_rstat, cgroup_rstat_locked,

        TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),

        TP_ARGS(cgrp, cpu, contended)
);

DEFINE_EVENT(cgroup_rstat, cgroup_rstat_unlock,

        TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),

        TP_ARGS(cgrp, cpu, contended)
);

/* Related to per CPU: cgroup_rstat_cpu_lock */
DEFINE_EVENT(cgroup_rstat, cgroup_rstat_cpu_lock_contended,

        TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),

        TP_ARGS(cgrp, cpu, contended)
);

DEFINE_EVENT(cgroup_rstat, cgroup_rstat_cpu_lock_contended_fastpath,

        TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),

        TP_ARGS(cgrp, cpu, contended)
);

DEFINE_EVENT(cgroup_rstat, cgroup_rstat_cpu_locked,

        TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),

        TP_ARGS(cgrp, cpu, contended)
);

DEFINE_EVENT(cgroup_rstat, cgroup_rstat_cpu_locked_fastpath,

        TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),

        TP_ARGS(cgrp, cpu, contended)
);

DEFINE_EVENT(cgroup_rstat, cgroup_rstat_cpu_unlock,

        TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),

        TP_ARGS(cgrp, cpu, contended)
);

DEFINE_EVENT(cgroup_rstat, cgroup_rstat_cpu_unlock_fastpath,

        TP_PROTO(struct cgroup *cgrp, int cpu, bool contended),

        TP_ARGS(cgrp, cpu, contended)
);

#endif /* _TRACE_CGROUP_H */

/* This part must be outside protection */
#include <trace/define_trace.h>











































































































  212 


























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
// SPDX-License-Identifier: GPL-2.0
/* User-mappable watch queue
 *
 * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 *
 * See Documentation/core-api/watch_queue.rst
 */

#ifndef _LINUX_WATCH_QUEUE_H
#define _LINUX_WATCH_QUEUE_H

#include <uapi/linux/watch_queue.h>
#include <linux/kref.h>
#include <linux/rcupdate.h>

#ifdef CONFIG_WATCH_QUEUE

struct cred;

struct watch_type_filter {
        enum watch_notification_type type;
        __u32                subtype_filter[1];        /* Bitmask of subtypes to filter on */
        __u32                info_filter;                /* Filter on watch_notification::info */
        __u32                info_mask;                /* Mask of relevant bits in info_filter */
};

struct watch_filter {
        union {
                struct rcu_head        rcu;
                /* Bitmask of accepted types */
                DECLARE_BITMAP(type_filter, WATCH_TYPE__NR);
        };
        u32                        nr_filters;        /* Number of filters */
        struct watch_type_filter filters[] __counted_by(nr_filters);
};

struct watch_queue {
        struct rcu_head                rcu;
        struct watch_filter __rcu *filter;
        struct pipe_inode_info        *pipe;                /* Pipe we use as a buffer, NULL if queue closed */
        struct hlist_head        watches;        /* Contributory watches */
        struct page                **notes;        /* Preallocated notifications */
        unsigned long                *notes_bitmap;        /* Allocation bitmap for notes */
        struct kref                usage;                /* Object usage count */
        spinlock_t                lock;
        unsigned int                nr_notes;        /* Number of notes */
        unsigned int                nr_pages;        /* Number of pages in notes[] */
};

/*
 * Representation of a watch on an object.
 */
struct watch {
        union {
                struct rcu_head        rcu;
                u32                info_id;        /* ID to be OR'd in to info field */
        };
        struct watch_queue __rcu *queue;        /* Queue to post events to */
        struct hlist_node        queue_node;        /* Link in queue->watches */
        struct watch_list __rcu        *watch_list;
        struct hlist_node        list_node;        /* Link in watch_list->watchers */
        const struct cred        *cred;                /* Creds of the owner of the watch */
        void                        *private;        /* Private data for the watched object */
        u64                        id;                /* Internal identifier */
        struct kref                usage;                /* Object usage count */
};

/*
 * List of watches on an object.
 */
struct watch_list {
        struct rcu_head                rcu;
        struct hlist_head        watchers;
        void (*release_watch)(struct watch *);
        spinlock_t                lock;
};

extern void __post_watch_notification(struct watch_list *,
                                      struct watch_notification *,
                                      const struct cred *,
                                      u64);
extern struct watch_queue *get_watch_queue(int);
extern void put_watch_queue(struct watch_queue *);
extern void init_watch(struct watch *, struct watch_queue *);
extern int add_watch_to_object(struct watch *, struct watch_list *);
extern int remove_watch_from_object(struct watch_list *, struct watch_queue *, u64, bool);
extern long watch_queue_set_size(struct pipe_inode_info *, unsigned int);
extern long watch_queue_set_filter(struct pipe_inode_info *,
                                   struct watch_notification_filter __user *);
extern int watch_queue_init(struct pipe_inode_info *);
extern void watch_queue_clear(struct watch_queue *);

static inline void init_watch_list(struct watch_list *wlist,
                                   void (*release_watch)(struct watch *))
{
        INIT_HLIST_HEAD(&wlist->watchers);
        spin_lock_init(&wlist->lock);
        wlist->release_watch = release_watch;
}

static inline void post_watch_notification(struct watch_list *wlist,
                                           struct watch_notification *n,
                                           const struct cred *cred,
                                           u64 id)
{
        if (unlikely(wlist))
                __post_watch_notification(wlist, n, cred, id);
}

static inline void remove_watch_list(struct watch_list *wlist, u64 id)
{
        if (wlist) {
                remove_watch_from_object(wlist, NULL, id, true);
                kfree_rcu(wlist, rcu);
        }
}

/**
 * watch_sizeof - Calculate the information part of the size of a watch record,
 * given the structure size.
 */
#define watch_sizeof(STRUCT) (sizeof(STRUCT) << WATCH_INFO_LENGTH__SHIFT)

#else
static inline int watch_queue_init(struct pipe_inode_info *pipe)
{
        return -ENOPKG;
}

#endif

#endif /* _LINUX_WATCH_QUEUE_H */





















































   23 






   23 



























  103 





  103 









  103 





  103 





























   63 










   63 










































































































































































































































   40 
































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * kernel/stop_machine.c
 *
 * Copyright (C) 2008, 2005        IBM Corporation.
 * Copyright (C) 2008, 2005        Rusty Russell rusty@rustcorp.com.au
 * Copyright (C) 2010                SUSE Linux Products GmbH
 * Copyright (C) 2010                Tejun Heo <tj@kernel.org>
 */
#include <linux/compiler.h>
#include <linux/completion.h>
#include <linux/cpu.h>
#include <linux/init.h>
#include <linux/kthread.h>
#include <linux/export.h>
#include <linux/percpu.h>
#include <linux/sched.h>
#include <linux/stop_machine.h>
#include <linux/interrupt.h>
#include <linux/kallsyms.h>
#include <linux/smpboot.h>
#include <linux/atomic.h>
#include <linux/nmi.h>
#include <linux/sched/wake_q.h>

/*
 * Structure to determine completion condition and record errors.  May
 * be shared by works on different cpus.
 */
struct cpu_stop_done {
        atomic_t                nr_todo;        /* nr left to execute */
        int                        ret;                /* collected return value */
        struct completion        completion;        /* fired if nr_todo reaches 0 */
};

/* the actual stopper, one per every possible cpu, enabled on online cpus */
struct cpu_stopper {
        struct task_struct        *thread;

        raw_spinlock_t                lock;
        bool                        enabled;        /* is this stopper enabled? */
        struct list_head        works;                /* list of pending works */

        struct cpu_stop_work        stop_work;        /* for stop_cpus */
        unsigned long                caller;
        cpu_stop_fn_t                fn;
};

static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
static bool stop_machine_initialized = false;

void print_stop_info(const char *log_lvl, struct task_struct *task)
{
        /*
         * If @task is a stopper task, it cannot migrate and task_cpu() is
         * stable.
         */
        struct cpu_stopper *stopper = per_cpu_ptr(&cpu_stopper, task_cpu(task));

        if (task != stopper->thread)
                return;

        printk("%sStopper: %pS <- %pS\n", log_lvl, stopper->fn, (void *)stopper->caller);
}

/* static data for stop_cpus */
static DEFINE_MUTEX(stop_cpus_mutex);
static bool stop_cpus_in_progress;

static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
{
        memset(done, 0, sizeof(*done));
        atomic_set(&done->nr_todo, nr_todo);
        init_completion(&done->completion);
}

/* signal completion unless @done is NULL */
static void cpu_stop_signal_done(struct cpu_stop_done *done)
{
        if (atomic_dec_and_test(&done->nr_todo))
                complete(&done->completion);
}

static void __cpu_stop_queue_work(struct cpu_stopper *stopper,
                                        struct cpu_stop_work *work,
                                        struct wake_q_head *wakeq)
{
        list_add_tail(&work->list, &stopper->works);
        wake_q_add(wakeq, stopper->thread);
}

/* queue @work to @stopper.  if offline, @work is completed immediately */
static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work)
{
        struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
        DEFINE_WAKE_Q(wakeq);
        unsigned long flags;
        bool enabled;

        preempt_disable();
        raw_spin_lock_irqsave(&stopper->lock, flags);
        enabled = stopper->enabled;
        if (enabled)
                __cpu_stop_queue_work(stopper, work, &wakeq);
        else if (work->done)
                cpu_stop_signal_done(work->done);
        raw_spin_unlock_irqrestore(&stopper->lock, flags);

        wake_up_q(&wakeq);
        preempt_enable();

        return enabled;
}

/**
 * stop_one_cpu - stop a cpu
 * @cpu: cpu to stop
 * @fn: function to execute
 * @arg: argument to @fn
 *
 * Execute @fn(@arg) on @cpu.  @fn is run in a process context with
 * the highest priority preempting any task on the cpu and
 * monopolizing it.  This function returns after the execution is
 * complete.
 *
 * This function doesn't guarantee @cpu stays online till @fn
 * completes.  If @cpu goes down in the middle, execution may happen
 * partially or fully on different cpus.  @fn should either be ready
 * for that or the caller should ensure that @cpu stays online until
 * this function completes.
 *
 * CONTEXT:
 * Might sleep.
 *
 * RETURNS:
 * -ENOENT if @fn(@arg) was not executed because @cpu was offline;
 * otherwise, the return value of @fn.
 */
int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
{
        struct cpu_stop_done done;
        struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done, .caller = _RET_IP_ };

        cpu_stop_init_done(&done, 1);
        if (!cpu_stop_queue_work(cpu, &work))
                return -ENOENT;
        /*
         * In case @cpu == smp_proccessor_id() we can avoid a sleep+wakeup
         * cycle by doing a preemption:
         */
        cond_resched();
        wait_for_completion(&done.completion);
        return done.ret;
}

/* This controls the threads on each CPU. */
enum multi_stop_state {
        /* Dummy starting state for thread. */
        MULTI_STOP_NONE,
        /* Awaiting everyone to be scheduled. */
        MULTI_STOP_PREPARE,
        /* Disable interrupts. */
        MULTI_STOP_DISABLE_IRQ,
        /* Run the function */
        MULTI_STOP_RUN,
        /* Exit */
        MULTI_STOP_EXIT,
};

struct multi_stop_data {
        cpu_stop_fn_t                fn;
        void                        *data;
        /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
        unsigned int                num_threads;
        const struct cpumask        *active_cpus;

        enum multi_stop_state        state;
        atomic_t                thread_ack;
};

static void set_state(struct multi_stop_data *msdata,
                      enum multi_stop_state newstate)
{
        /* Reset ack counter. */
        atomic_set(&msdata->thread_ack, msdata->num_threads);
        smp_wmb();
        WRITE_ONCE(msdata->state, newstate);
}

/* Last one to ack a state moves to the next state. */
static void ack_state(struct multi_stop_data *msdata)
{
        if (atomic_dec_and_test(&msdata->thread_ack))
                set_state(msdata, msdata->state + 1);
}

notrace void __weak stop_machine_yield(const struct cpumask *cpumask)
{
        cpu_relax();
}

/* This is the cpu_stop function which stops the CPU. */
static int multi_cpu_stop(void *data)
{
        struct multi_stop_data *msdata = data;
        enum multi_stop_state newstate, curstate = MULTI_STOP_NONE;
        int cpu = smp_processor_id(), err = 0;
        const struct cpumask *cpumask;
        unsigned long flags;
        bool is_active;

        /*
         * When called from stop_machine_from_inactive_cpu(), irq might
         * already be disabled.  Save the state and restore it on exit.
         */
        local_save_flags(flags);

        if (!msdata->active_cpus) {
                cpumask = cpu_online_mask;
                is_active = cpu == cpumask_first(cpumask);
        } else {
                cpumask = msdata->active_cpus;
                is_active = cpumask_test_cpu(cpu, cpumask);
        }

        /* Simple state machine */
        do {
                /* Chill out and ensure we re-read multi_stop_state. */
                stop_machine_yield(cpumask);
                newstate = READ_ONCE(msdata->state);
                if (newstate != curstate) {
                        curstate = newstate;
                        switch (curstate) {
                        case MULTI_STOP_DISABLE_IRQ:
                                local_irq_disable();
                                hard_irq_disable();
                                break;
                        case MULTI_STOP_RUN:
                                if (is_active)
                                        err = msdata->fn(msdata->data);
                                break;
                        default:
                                break;
                        }
                        ack_state(msdata);
                } else if (curstate > MULTI_STOP_PREPARE) {
                        /*
                         * At this stage all other CPUs we depend on must spin
                         * in the same loop. Any reason for hard-lockup should
                         * be detected and reported on their side.
                         */
                        touch_nmi_watchdog();
                }
                rcu_momentary_eqs();
        } while (curstate != MULTI_STOP_EXIT);

        local_irq_restore(flags);
        return err;
}

static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1,
                                    int cpu2, struct cpu_stop_work *work2)
{
        struct cpu_stopper *stopper1 = per_cpu_ptr(&cpu_stopper, cpu1);
        struct cpu_stopper *stopper2 = per_cpu_ptr(&cpu_stopper, cpu2);
        DEFINE_WAKE_Q(wakeq);
        int err;

retry:
        /*
         * The waking up of stopper threads has to happen in the same
         * scheduling context as the queueing.  Otherwise, there is a
         * possibility of one of the above stoppers being woken up by another
         * CPU, and preempting us. This will cause us to not wake up the other
         * stopper forever.
         */
        preempt_disable();
        raw_spin_lock_irq(&stopper1->lock);
        raw_spin_lock_nested(&stopper2->lock, SINGLE_DEPTH_NESTING);

        if (!stopper1->enabled || !stopper2->enabled) {
                err = -ENOENT;
                goto unlock;
        }

        /*
         * Ensure that if we race with __stop_cpus() the stoppers won't get
         * queued up in reverse order leading to system deadlock.
         *
         * We can't miss stop_cpus_in_progress if queue_stop_cpus_work() has
         * queued a work on cpu1 but not on cpu2, we hold both locks.
         *
         * It can be falsely true but it is safe to spin until it is cleared,
         * queue_stop_cpus_work() does everything under preempt_disable().
         */
        if (unlikely(stop_cpus_in_progress)) {
                err = -EDEADLK;
                goto unlock;
        }

        err = 0;
        __cpu_stop_queue_work(stopper1, work1, &wakeq);
        __cpu_stop_queue_work(stopper2, work2, &wakeq);

unlock:
        raw_spin_unlock(&stopper2->lock);
        raw_spin_unlock_irq(&stopper1->lock);

        if (unlikely(err == -EDEADLK)) {
                preempt_enable();

                while (stop_cpus_in_progress)
                        cpu_relax();

                goto retry;
        }

        wake_up_q(&wakeq);
        preempt_enable();

        return err;
}
/**
 * stop_two_cpus - stops two cpus
 * @cpu1: the cpu to stop
 * @cpu2: the other cpu to stop
 * @fn: function to execute
 * @arg: argument to @fn
 *
 * Stops both the current and specified CPU and runs @fn on one of them.
 *
 * returns when both are completed.
 */
int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg)
{
        struct cpu_stop_done done;
        struct cpu_stop_work work1, work2;
        struct multi_stop_data msdata;

        msdata = (struct multi_stop_data){
                .fn = fn,
                .data = arg,
                .num_threads = 2,
                .active_cpus = cpumask_of(cpu1),
        };

        work1 = work2 = (struct cpu_stop_work){
                .fn = multi_cpu_stop,
                .arg = &msdata,
                .done = &done,
                .caller = _RET_IP_,
        };

        cpu_stop_init_done(&done, 2);
        set_state(&msdata, MULTI_STOP_PREPARE);

        if (cpu1 > cpu2)
                swap(cpu1, cpu2);
        if (cpu_stop_queue_two_works(cpu1, &work1, cpu2, &work2))
                return -ENOENT;

        wait_for_completion(&done.completion);
        return done.ret;
}

/**
 * stop_one_cpu_nowait - stop a cpu but don't wait for completion
 * @cpu: cpu to stop
 * @fn: function to execute
 * @arg: argument to @fn
 * @work_buf: pointer to cpu_stop_work structure
 *
 * Similar to stop_one_cpu() but doesn't wait for completion.  The
 * caller is responsible for ensuring @work_buf is currently unused
 * and will remain untouched until stopper starts executing @fn.
 *
 * CONTEXT:
 * Don't care.
 *
 * RETURNS:
 * true if cpu_stop_work was queued successfully and @fn will be called,
 * false otherwise.
 */
bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
                        struct cpu_stop_work *work_buf)
{
        *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, .caller = _RET_IP_, };
        return cpu_stop_queue_work(cpu, work_buf);
}

static bool queue_stop_cpus_work(const struct cpumask *cpumask,
                                 cpu_stop_fn_t fn, void *arg,
                                 struct cpu_stop_done *done)
{
        struct cpu_stop_work *work;
        unsigned int cpu;
        bool queued = false;

        /*
         * Disable preemption while queueing to avoid getting
         * preempted by a stopper which might wait for other stoppers
         * to enter @fn which can lead to deadlock.
         */
        preempt_disable();
        stop_cpus_in_progress = true;
        barrier();
        for_each_cpu(cpu, cpumask) {
                work = &per_cpu(cpu_stopper.stop_work, cpu);
                work->fn = fn;
                work->arg = arg;
                work->done = done;
                work->caller = _RET_IP_;
                if (cpu_stop_queue_work(cpu, work))
                        queued = true;
        }
        barrier();
        stop_cpus_in_progress = false;
        preempt_enable();

        return queued;
}

static int __stop_cpus(const struct cpumask *cpumask,
                       cpu_stop_fn_t fn, void *arg)
{
        struct cpu_stop_done done;

        cpu_stop_init_done(&done, cpumask_weight(cpumask));
        if (!queue_stop_cpus_work(cpumask, fn, arg, &done))
                return -ENOENT;
        wait_for_completion(&done.completion);
        return done.ret;
}

/**
 * stop_cpus - stop multiple cpus
 * @cpumask: cpus to stop
 * @fn: function to execute
 * @arg: argument to @fn
 *
 * Execute @fn(@arg) on online cpus in @cpumask.  On each target cpu,
 * @fn is run in a process context with the highest priority
 * preempting any task on the cpu and monopolizing it.  This function
 * returns after all executions are complete.
 *
 * This function doesn't guarantee the cpus in @cpumask stay online
 * till @fn completes.  If some cpus go down in the middle, execution
 * on the cpu may happen partially or fully on different cpus.  @fn
 * should either be ready for that or the caller should ensure that
 * the cpus stay online until this function completes.
 *
 * All stop_cpus() calls are serialized making it safe for @fn to wait
 * for all cpus to start executing it.
 *
 * CONTEXT:
 * Might sleep.
 *
 * RETURNS:
 * -ENOENT if @fn(@arg) was not executed at all because all cpus in
 * @cpumask were offline; otherwise, 0 if all executions of @fn
 * returned 0, any non zero return value if any returned non zero.
 */
static int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
{
        int ret;

        /* static works are used, process one request at a time */
        mutex_lock(&stop_cpus_mutex);
        ret = __stop_cpus(cpumask, fn, arg);
        mutex_unlock(&stop_cpus_mutex);
        return ret;
}

static int cpu_stop_should_run(unsigned int cpu)
{
        struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
        unsigned long flags;
        int run;

        raw_spin_lock_irqsave(&stopper->lock, flags);
        run = !list_empty(&stopper->works);
        raw_spin_unlock_irqrestore(&stopper->lock, flags);
        return run;
}

static void cpu_stopper_thread(unsigned int cpu)
{
        struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
        struct cpu_stop_work *work;

repeat:
        work = NULL;
        raw_spin_lock_irq(&stopper->lock);
        if (!list_empty(&stopper->works)) {
                work = list_first_entry(&stopper->works,
                                        struct cpu_stop_work, list);
                list_del_init(&work->list);
        }
        raw_spin_unlock_irq(&stopper->lock);

        if (work) {
                cpu_stop_fn_t fn = work->fn;
                void *arg = work->arg;
                struct cpu_stop_done *done = work->done;
                int ret;

                /* cpu stop callbacks must not sleep, make in_atomic() == T */
                stopper->caller = work->caller;
                stopper->fn = fn;
                preempt_count_inc();
                ret = fn(arg);
                if (done) {
                        if (ret)
                                done->ret = ret;
                        cpu_stop_signal_done(done);
                }
                preempt_count_dec();
                stopper->fn = NULL;
                stopper->caller = 0;
                WARN_ONCE(preempt_count(),
                          "cpu_stop: %ps(%p) leaked preempt count\n", fn, arg);
                goto repeat;
        }
}

void stop_machine_park(int cpu)
{
        struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
        /*
         * Lockless. cpu_stopper_thread() will take stopper->lock and flush
         * the pending works before it parks, until then it is fine to queue
         * the new works.
         */
        stopper->enabled = false;
        kthread_park(stopper->thread);
}

static void cpu_stop_create(unsigned int cpu)
{
        sched_set_stop_task(cpu, per_cpu(cpu_stopper.thread, cpu));
}

static void cpu_stop_park(unsigned int cpu)
{
        struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);

        WARN_ON(!list_empty(&stopper->works));
}

void stop_machine_unpark(int cpu)
{
        struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);

        stopper->enabled = true;
        kthread_unpark(stopper->thread);
}

static struct smp_hotplug_thread cpu_stop_threads = {
        .store                        = &cpu_stopper.thread,
        .thread_should_run        = cpu_stop_should_run,
        .thread_fn                = cpu_stopper_thread,
        .thread_comm                = "migration/%u",
        .create                        = cpu_stop_create,
        .park                        = cpu_stop_park,
        .selfparking                = true,
};

static int __init cpu_stop_init(void)
{
        unsigned int cpu;

        for_each_possible_cpu(cpu) {
                struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);

                raw_spin_lock_init(&stopper->lock);
                INIT_LIST_HEAD(&stopper->works);
        }

        BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
        stop_machine_unpark(raw_smp_processor_id());
        stop_machine_initialized = true;
        return 0;
}
early_initcall(cpu_stop_init);

int stop_machine_cpuslocked(cpu_stop_fn_t fn, void *data,
                            const struct cpumask *cpus)
{
        struct multi_stop_data msdata = {
                .fn = fn,
                .data = data,
                .num_threads = num_online_cpus(),
                .active_cpus = cpus,
        };

        lockdep_assert_cpus_held();

        if (!stop_machine_initialized) {
                /*
                 * Handle the case where stop_machine() is called
                 * early in boot before stop_machine() has been
                 * initialized.
                 */
                unsigned long flags;
                int ret;

                WARN_ON_ONCE(msdata.num_threads != 1);

                local_irq_save(flags);
                hard_irq_disable();
                ret = (*fn)(data);
                local_irq_restore(flags);

                return ret;
        }

        /* Set the initial state and stop all online cpus. */
        set_state(&msdata, MULTI_STOP_PREPARE);
        return stop_cpus(cpu_online_mask, multi_cpu_stop, &msdata);
}

int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus)
{
        int ret;

        /* No CPUs can come up or down during this. */
        cpus_read_lock();
        ret = stop_machine_cpuslocked(fn, data, cpus);
        cpus_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(stop_machine);

#ifdef CONFIG_SCHED_SMT
int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data)
{
        const struct cpumask *smt_mask = cpu_smt_mask(cpu);

        struct multi_stop_data msdata = {
                .fn = fn,
                .data = data,
                .num_threads = cpumask_weight(smt_mask),
                .active_cpus = smt_mask,
        };

        lockdep_assert_cpus_held();

        /* Set the initial state and stop all online cpus. */
        set_state(&msdata, MULTI_STOP_PREPARE);
        return stop_cpus(smt_mask, multi_cpu_stop, &msdata);
}
EXPORT_SYMBOL_GPL(stop_core_cpuslocked);
#endif

/**
 * stop_machine_from_inactive_cpu - stop_machine() from inactive CPU
 * @fn: the function to run
 * @data: the data ptr for the @fn()
 * @cpus: the cpus to run the @fn() on (NULL = any online cpu)
 *
 * This is identical to stop_machine() but can be called from a CPU which
 * is not active.  The local CPU is in the process of hotplug (so no other
 * CPU hotplug can start) and not marked active and doesn't have enough
 * context to sleep.
 *
 * This function provides stop_machine() functionality for such state by
 * using busy-wait for synchronization and executing @fn directly for local
 * CPU.
 *
 * CONTEXT:
 * Local CPU is inactive.  Temporarily stops all active CPUs.
 *
 * RETURNS:
 * 0 if all executions of @fn returned 0, any non zero return value if any
 * returned non zero.
 */
int stop_machine_from_inactive_cpu(cpu_stop_fn_t fn, void *data,
                                  const struct cpumask *cpus)
{
        struct multi_stop_data msdata = { .fn = fn, .data = data,
                                            .active_cpus = cpus };
        struct cpu_stop_done done;
        int ret;

        /* Local CPU must be inactive and CPU hotplug in progress. */
        BUG_ON(cpu_active(raw_smp_processor_id()));
        msdata.num_threads = num_active_cpus() + 1;        /* +1 for local */

        /* No proper task established and can't sleep - busy wait for lock. */
        while (!mutex_trylock(&stop_cpus_mutex))
                cpu_relax();

        /* Schedule work on other CPUs and execute directly for local CPU */
        set_state(&msdata, MULTI_STOP_PREPARE);
        cpu_stop_init_done(&done, num_active_cpus());
        queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata,
                             &done);
        ret = multi_cpu_stop(&msdata);

        /* Busy wait for completion. */
        while (!completion_done(&done.completion))
                cpu_relax();

        mutex_unlock(&stop_cpus_mutex);
        return ret ?: done.ret;
}


















































































































































































 39091 












  604 



























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _ASM_X86_CPUFEATURE_H
#define _ASM_X86_CPUFEATURE_H

#include <asm/processor.h>

#if defined(__KERNEL__) && !defined(__ASSEMBLY__)

#include <asm/asm.h>
#include <linux/bitops.h>
#include <asm/alternative.h>

enum cpuid_leafs
{
        CPUID_1_EDX                = 0,
        CPUID_8000_0001_EDX,
        CPUID_8086_0001_EDX,
        CPUID_LNX_1,
        CPUID_1_ECX,
        CPUID_C000_0001_EDX,
        CPUID_8000_0001_ECX,
        CPUID_LNX_2,
        CPUID_LNX_3,
        CPUID_7_0_EBX,
        CPUID_D_1_EAX,
        CPUID_LNX_4,
        CPUID_7_1_EAX,
        CPUID_8000_0008_EBX,
        CPUID_6_EAX,
        CPUID_8000_000A_EDX,
        CPUID_7_ECX,
        CPUID_8000_0007_EBX,
        CPUID_7_EDX,
        CPUID_8000_001F_EAX,
        CPUID_8000_0021_EAX,
        CPUID_LNX_5,
        NR_CPUID_WORDS,
};

#define X86_CAP_FMT_NUM "%d:%d"
#define x86_cap_flag_num(flag) ((flag) >> 5), ((flag) & 31)

extern const char * const x86_cap_flags[NCAPINTS*32];
extern const char * const x86_power_flags[32];
#define X86_CAP_FMT "%s"
#define x86_cap_flag(flag) x86_cap_flags[flag]

/*
 * In order to save room, we index into this array by doing
 * X86_BUG_<name> - NCAPINTS*32.
 */
extern const char * const x86_bug_flags[NBUGINTS*32];

#define test_cpu_cap(c, bit)                                                \
         arch_test_bit(bit, (unsigned long *)((c)->x86_capability))

/*
 * There are 32 bits/features in each mask word.  The high bits
 * (selected with (bit>>5) give us the word number and the low 5
 * bits give us the bit/feature number inside the word.
 * (1UL<<((bit)&31) gives us a mask for the feature_bit so we can
 * see if it is set in the mask word.
 */
#define CHECK_BIT_IN_MASK_WORD(maskname, word, bit)        \
        (((bit)>>5)==(word) && (1UL<<((bit)&31) & maskname##word ))

/*
 * {REQUIRED,DISABLED}_MASK_CHECK below may seem duplicated with the
 * following BUILD_BUG_ON_ZERO() check but when NCAPINTS gets changed, all
 * header macros which use NCAPINTS need to be changed. The duplicated macro
 * use causes the compiler to issue errors for all headers so that all usage
 * sites can be corrected.
 */
#define REQUIRED_MASK_BIT_SET(feature_bit)                \
         ( CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  0, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  1, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  2, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  3, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  4, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  5, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  6, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  7, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  8, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK,  9, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 10, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 11, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 12, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 13, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 14, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 15, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 16, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 17, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 18, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 19, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 20, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(REQUIRED_MASK, 21, feature_bit) ||        \
           REQUIRED_MASK_CHECK                                          ||        \
           BUILD_BUG_ON_ZERO(NCAPINTS != 22))

#define DISABLED_MASK_BIT_SET(feature_bit)                                \
         ( CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  0, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  1, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  2, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  3, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  4, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  5, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  6, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  7, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  8, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK,  9, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 10, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 11, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 12, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 13, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 14, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 15, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 16, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 17, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 18, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 19, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 20, feature_bit) ||        \
           CHECK_BIT_IN_MASK_WORD(DISABLED_MASK, 21, feature_bit) ||        \
           DISABLED_MASK_CHECK                                          ||        \
           BUILD_BUG_ON_ZERO(NCAPINTS != 22))

#define cpu_has(c, bit)                                                        \
        (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :        \
         test_cpu_cap(c, bit))

#define this_cpu_has(bit)                                                \
        (__builtin_constant_p(bit) && REQUIRED_MASK_BIT_SET(bit) ? 1 :        \
         x86_this_cpu_test_bit(bit, cpu_info.x86_capability))

/*
 * This macro is for detection of features which need kernel
 * infrastructure to be used.  It may *not* directly test the CPU
 * itself.  Use the cpu_has() family if you want true runtime
 * testing of CPU features, like in hypervisor code where you are
 * supporting a possible guest feature where host support for it
 * is not relevant.
 */
#define cpu_feature_enabled(bit)        \
        (__builtin_constant_p(bit) && DISABLED_MASK_BIT_SET(bit) ? 0 : static_cpu_has(bit))

#define boot_cpu_has(bit)        cpu_has(&boot_cpu_data, bit)

#define set_cpu_cap(c, bit)        set_bit(bit, (unsigned long *)((c)->x86_capability))

extern void setup_clear_cpu_cap(unsigned int bit);
extern void clear_cpu_cap(struct cpuinfo_x86 *c, unsigned int bit);

#define setup_force_cpu_cap(bit) do {                        \
                                                        \
        if (!boot_cpu_has(bit))                                \
                WARN_ON(alternatives_patched);                \
                                                        \
        set_cpu_cap(&boot_cpu_data, bit);                \
        set_bit(bit, (unsigned long *)cpu_caps_set);        \
} while (0)

#define setup_force_cpu_bug(bit) setup_force_cpu_cap(bit)

/*
 * Static testing of CPU features. Used the same as boot_cpu_has(). It
 * statically patches the target code for additional performance. Use
 * static_cpu_has() only in fast paths, where every cycle counts. Which
 * means that the boot_cpu_has() variant is already fast enough for the
 * majority of cases and you should stick to using it as it is generally
 * only two instructions: a RIP-relative MOV and a TEST.
 *
 * Do not use an "m" constraint for [cap_byte] here: gcc doesn't know
 * that this is only used on a fallback path and will sometimes cause
 * it to manifest the address of boot_cpu_data in a register, fouling
 * the mainline (post-initialization) code.
 */
static __always_inline bool _static_cpu_has(u16 bit)
{
        asm goto(ALTERNATIVE_TERNARY("jmp 6f", %c[feature], "", "jmp %l[t_no]")
                ".pushsection .altinstr_aux,\"ax\"\n"
                "6:\n"
                " testb %[bitnum], %a[cap_byte]\n"
                " jnz %l[t_yes]\n"
                " jmp %l[t_no]\n"
                ".popsection\n"
                 : : [feature]  "i" (bit),
                     [bitnum]   "i" (1 << (bit & 7)),
                     [cap_byte] "i" (&((const char *)boot_cpu_data.x86_capability)[bit >> 3])
                 : : t_yes, t_no);
t_yes:
        return true;
t_no:
        return false;
}

#define static_cpu_has(bit)                                        \
(                                                                \
        __builtin_constant_p(boot_cpu_has(bit)) ?                \
                boot_cpu_has(bit) :                                \
                _static_cpu_has(bit)                                \
)

#define cpu_has_bug(c, bit)                cpu_has(c, (bit))
#define set_cpu_bug(c, bit)                set_cpu_cap(c, (bit))
#define clear_cpu_bug(c, bit)                clear_cpu_cap(c, (bit))

#define static_cpu_has_bug(bit)                static_cpu_has((bit))
#define boot_cpu_has_bug(bit)                cpu_has_bug(&boot_cpu_data, (bit))
#define boot_cpu_set_bug(bit)                set_cpu_cap(&boot_cpu_data, (bit))

#define MAX_CPU_FEATURES                (NCAPINTS * 32)
#define cpu_have_feature                boot_cpu_has

#define CPU_FEATURE_TYPEFMT                "x86,ven%04Xfam%04Xmod%04X"
#define CPU_FEATURE_TYPEVAL                boot_cpu_data.x86_vendor, boot_cpu_data.x86, \
                                        boot_cpu_data.x86_model

#endif /* defined(__KERNEL__) && !defined(__ASSEMBLY__) */
#endif /* _ASM_X86_CPUFEATURE_H */














































  837 
   50 








   22 
  136 














































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
/* SPDX-License-Identifier: GPL-2.0 */
/*
  File: linux/posix_acl.h

  (C) 2002 Andreas Gruenbacher, <a.gruenbacher@computer.org>
*/


#ifndef __LINUX_POSIX_ACL_H
#define __LINUX_POSIX_ACL_H

#include <linux/bug.h>
#include <linux/slab.h>
#include <linux/rcupdate.h>
#include <linux/refcount.h>
#include <uapi/linux/posix_acl.h>

struct user_namespace;

struct posix_acl_entry {
        short                        e_tag;
        unsigned short                e_perm;
        union {
                kuid_t                e_uid;
                kgid_t                e_gid;
        };
};

struct posix_acl {
        refcount_t                a_refcount;
        struct rcu_head                a_rcu;
        unsigned int                a_count;
        struct posix_acl_entry        a_entries[];
};

#define FOREACH_ACL_ENTRY(pa, acl, pe) \
        for(pa=(acl)->a_entries, pe=pa+(acl)->a_count; pa<pe; pa++)


/*
 * Duplicate an ACL handle.
 */
static inline struct posix_acl *
posix_acl_dup(struct posix_acl *acl)
{
        if (acl)
                refcount_inc(&acl->a_refcount);
        return acl;
}

/*
 * Free an ACL handle.
 */
static inline void
posix_acl_release(struct posix_acl *acl)
{
        if (acl && refcount_dec_and_test(&acl->a_refcount))
                kfree_rcu(acl, a_rcu);
}


/* posix_acl.c */

extern void posix_acl_init(struct posix_acl *, int);
extern struct posix_acl *posix_acl_alloc(int, gfp_t);
extern struct posix_acl *posix_acl_from_mode(umode_t, gfp_t);
extern int posix_acl_equiv_mode(const struct posix_acl *, umode_t *);
extern int __posix_acl_create(struct posix_acl **, gfp_t, umode_t *);
extern int __posix_acl_chmod(struct posix_acl **, gfp_t, umode_t);

extern struct posix_acl *get_posix_acl(struct inode *, int);
int set_posix_acl(struct mnt_idmap *, struct dentry *, int,
                  struct posix_acl *);

struct posix_acl *get_cached_acl_rcu(struct inode *inode, int type);
struct posix_acl *posix_acl_clone(const struct posix_acl *acl, gfp_t flags);

#ifdef CONFIG_FS_POSIX_ACL
int posix_acl_chmod(struct mnt_idmap *, struct dentry *, umode_t);
extern int posix_acl_create(struct inode *, umode_t *, struct posix_acl **,
                struct posix_acl **);
int posix_acl_update_mode(struct mnt_idmap *, struct inode *, umode_t *,
                          struct posix_acl **);

int simple_set_acl(struct mnt_idmap *, struct dentry *,
                   struct posix_acl *, int);
extern int simple_acl_create(struct inode *, struct inode *);

struct posix_acl *get_cached_acl(struct inode *inode, int type);
void set_cached_acl(struct inode *inode, int type, struct posix_acl *acl);
void forget_cached_acl(struct inode *inode, int type);
void forget_all_cached_acls(struct inode *inode);
int posix_acl_valid(struct user_namespace *, const struct posix_acl *);
int posix_acl_permission(struct mnt_idmap *, struct inode *,
                         const struct posix_acl *, int);

static inline void cache_no_acl(struct inode *inode)
{
        inode->i_acl = NULL;
        inode->i_default_acl = NULL;
}

int vfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry,
                const char *acl_name, struct posix_acl *kacl);
struct posix_acl *vfs_get_acl(struct mnt_idmap *idmap,
                              struct dentry *dentry, const char *acl_name);
int vfs_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry,
                   const char *acl_name);
int posix_acl_listxattr(struct inode *inode, char **buffer,
                        ssize_t *remaining_size);
#else
static inline int posix_acl_chmod(struct mnt_idmap *idmap,
                                  struct dentry *dentry, umode_t mode)
{
        return 0;
}

#define simple_set_acl                NULL

static inline int simple_acl_create(struct inode *dir, struct inode *inode)
{
        return 0;
}
static inline void cache_no_acl(struct inode *inode)
{
}

static inline int posix_acl_create(struct inode *inode, umode_t *mode,
                struct posix_acl **default_acl, struct posix_acl **acl)
{
        *default_acl = *acl = NULL;
        return 0;
}

static inline void forget_all_cached_acls(struct inode *inode)
{
}

static inline int vfs_set_acl(struct mnt_idmap *idmap,
                              struct dentry *dentry, const char *name,
                              struct posix_acl *acl)
{
        return -EOPNOTSUPP;
}

static inline struct posix_acl *vfs_get_acl(struct mnt_idmap *idmap,
                                            struct dentry *dentry,
                                            const char *acl_name)
{
        return ERR_PTR(-EOPNOTSUPP);
}

static inline int vfs_remove_acl(struct mnt_idmap *idmap,
                                 struct dentry *dentry, const char *acl_name)
{
        return -EOPNOTSUPP;
}
static inline int posix_acl_listxattr(struct inode *inode, char **buffer,
                                      ssize_t *remaining_size)
{
        return 0;
}
#endif /* CONFIG_FS_POSIX_ACL */

struct posix_acl *get_inode_acl(struct inode *inode, int type);

#endif  /* __LINUX_POSIX_ACL_H */








































































































































































































































































































































































































































































































































































































































   19 



   19 









   19 



    9 


   10 







































































































   19 









    9 

   10 






    9 
   10 






   17 

    2 












   17 
    2 













   19 


























    9 
























    9 














1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
// SPDX-License-Identifier: GPL-2.0
/*
 * Functions related to sysfs handling
 */
#include <linux/kernel.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
#include <linux/blktrace_api.h>
#include <linux/debugfs.h>

#include "blk.h"
#include "blk-mq.h"
#include "blk-mq-debugfs.h"
#include "blk-mq-sched.h"
#include "blk-rq-qos.h"
#include "blk-wbt.h"
#include "blk-cgroup.h"
#include "blk-throttle.h"

struct queue_sysfs_entry {
        struct attribute attr;
        ssize_t (*show)(struct gendisk *disk, char *page);
        int (*load_module)(struct gendisk *disk, const char *page, size_t count);
        ssize_t (*store)(struct gendisk *disk, const char *page, size_t count);
};

static ssize_t
queue_var_show(unsigned long var, char *page)
{
        return sprintf(page, "%lu\n", var);
}

static ssize_t
queue_var_store(unsigned long *var, const char *page, size_t count)
{
        int err;
        unsigned long v;

        err = kstrtoul(page, 10, &v);
        if (err || v > UINT_MAX)
                return -EINVAL;

        *var = v;

        return count;
}

static ssize_t queue_requests_show(struct gendisk *disk, char *page)
{
        return queue_var_show(disk->queue->nr_requests, page);
}

static ssize_t
queue_requests_store(struct gendisk *disk, const char *page, size_t count)
{
        unsigned long nr;
        int ret, err;

        if (!queue_is_mq(disk->queue))
                return -EINVAL;

        ret = queue_var_store(&nr, page, count);
        if (ret < 0)
                return ret;

        if (nr < BLKDEV_MIN_RQ)
                nr = BLKDEV_MIN_RQ;

        err = blk_mq_update_nr_requests(disk->queue, nr);
        if (err)
                return err;

        return ret;
}

static ssize_t queue_ra_show(struct gendisk *disk, char *page)
{
        return queue_var_show(disk->bdi->ra_pages << (PAGE_SHIFT - 10), page);
}

static ssize_t
queue_ra_store(struct gendisk *disk, const char *page, size_t count)
{
        unsigned long ra_kb;
        ssize_t ret;

        ret = queue_var_store(&ra_kb, page, count);
        if (ret < 0)
                return ret;
        disk->bdi->ra_pages = ra_kb >> (PAGE_SHIFT - 10);
        return ret;
}

#define QUEUE_SYSFS_LIMIT_SHOW(_field)                                        \
static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return queue_var_show(disk->queue->limits._field, page);        \
}

QUEUE_SYSFS_LIMIT_SHOW(max_segments)
QUEUE_SYSFS_LIMIT_SHOW(max_discard_segments)
QUEUE_SYSFS_LIMIT_SHOW(max_integrity_segments)
QUEUE_SYSFS_LIMIT_SHOW(max_segment_size)
QUEUE_SYSFS_LIMIT_SHOW(logical_block_size)
QUEUE_SYSFS_LIMIT_SHOW(physical_block_size)
QUEUE_SYSFS_LIMIT_SHOW(chunk_sectors)
QUEUE_SYSFS_LIMIT_SHOW(io_min)
QUEUE_SYSFS_LIMIT_SHOW(io_opt)
QUEUE_SYSFS_LIMIT_SHOW(discard_granularity)
QUEUE_SYSFS_LIMIT_SHOW(zone_write_granularity)
QUEUE_SYSFS_LIMIT_SHOW(virt_boundary_mask)
QUEUE_SYSFS_LIMIT_SHOW(dma_alignment)
QUEUE_SYSFS_LIMIT_SHOW(max_open_zones)
QUEUE_SYSFS_LIMIT_SHOW(max_active_zones)
QUEUE_SYSFS_LIMIT_SHOW(atomic_write_unit_min)
QUEUE_SYSFS_LIMIT_SHOW(atomic_write_unit_max)

#define QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(_field)                        \
static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return sprintf(page, "%llu\n",                                        \
                (unsigned long long)disk->queue->limits._field <<        \
                        SECTOR_SHIFT);                                        \
}

QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_discard_sectors)
QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_hw_discard_sectors)
QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(max_write_zeroes_sectors)
QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_max_sectors)
QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES(atomic_write_boundary_sectors)

#define QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(_field)                        \
static ssize_t queue_##_field##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return queue_var_show(disk->queue->limits._field >> 1, page);        \
}

QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(max_sectors)
QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_KB(max_hw_sectors)

#define QUEUE_SYSFS_SHOW_CONST(_name, _val)                                \
static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return sprintf(page, "%d\n", _val);                                \
}

/* deprecated fields */
QUEUE_SYSFS_SHOW_CONST(discard_zeroes_data, 0)
QUEUE_SYSFS_SHOW_CONST(write_same_max, 0)
QUEUE_SYSFS_SHOW_CONST(poll_delay, -1)

static ssize_t queue_max_discard_sectors_store(struct gendisk *disk,
                const char *page, size_t count)
{
        unsigned long max_discard_bytes;
        struct queue_limits lim;
        ssize_t ret;
        int err;

        ret = queue_var_store(&max_discard_bytes, page, count);
        if (ret < 0)
                return ret;

        if (max_discard_bytes & (disk->queue->limits.discard_granularity - 1))
                return -EINVAL;

        if ((max_discard_bytes >> SECTOR_SHIFT) > UINT_MAX)
                return -EINVAL;

        lim = queue_limits_start_update(disk->queue);
        lim.max_user_discard_sectors = max_discard_bytes >> SECTOR_SHIFT;
        err = queue_limits_commit_update(disk->queue, &lim);
        if (err)
                return err;
        return ret;
}

/*
 * For zone append queue_max_zone_append_sectors does not just return the
 * underlying queue limits, but actually contains a calculation.  Because of
 * that we can't simply use QUEUE_SYSFS_LIMIT_SHOW_SECTORS_TO_BYTES here.
 */
static ssize_t queue_zone_append_max_show(struct gendisk *disk, char *page)
{
        return sprintf(page, "%llu\n",
                (u64)queue_max_zone_append_sectors(disk->queue) <<
                        SECTOR_SHIFT);
}

static ssize_t
queue_max_sectors_store(struct gendisk *disk, const char *page, size_t count)
{
        unsigned long max_sectors_kb;
        struct queue_limits lim;
        ssize_t ret;
        int err;

        ret = queue_var_store(&max_sectors_kb, page, count);
        if (ret < 0)
                return ret;

        lim = queue_limits_start_update(disk->queue);
        lim.max_user_sectors = max_sectors_kb << 1;
        err = queue_limits_commit_update(disk->queue, &lim);
        if (err)
                return err;
        return ret;
}

static ssize_t queue_feature_store(struct gendisk *disk, const char *page,
                size_t count, blk_features_t feature)
{
        struct queue_limits lim;
        unsigned long val;
        ssize_t ret;

        ret = queue_var_store(&val, page, count);
        if (ret < 0)
                return ret;

        lim = queue_limits_start_update(disk->queue);
        if (val)
                lim.features |= feature;
        else
                lim.features &= ~feature;
        ret = queue_limits_commit_update(disk->queue, &lim);
        if (ret)
                return ret;
        return count;
}

#define QUEUE_SYSFS_FEATURE(_name, _feature)                                \
static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return sprintf(page, "%u\n",                                        \
                !!(disk->queue->limits.features & _feature));                \
}                                                                        \
static ssize_t queue_##_name##_store(struct gendisk *disk,                \
                const char *page, size_t count)                                \
{                                                                        \
        return queue_feature_store(disk, page, count, _feature);        \
}

QUEUE_SYSFS_FEATURE(rotational, BLK_FEAT_ROTATIONAL)
QUEUE_SYSFS_FEATURE(add_random, BLK_FEAT_ADD_RANDOM)
QUEUE_SYSFS_FEATURE(iostats, BLK_FEAT_IO_STAT)
QUEUE_SYSFS_FEATURE(stable_writes, BLK_FEAT_STABLE_WRITES);

#define QUEUE_SYSFS_FEATURE_SHOW(_name, _feature)                        \
static ssize_t queue_##_name##_show(struct gendisk *disk, char *page)        \
{                                                                        \
        return sprintf(page, "%u\n",                                        \
                !!(disk->queue->limits.features & _feature));                \
}

QUEUE_SYSFS_FEATURE_SHOW(poll, BLK_FEAT_POLL);
QUEUE_SYSFS_FEATURE_SHOW(fua, BLK_FEAT_FUA);
QUEUE_SYSFS_FEATURE_SHOW(dax, BLK_FEAT_DAX);

static ssize_t queue_zoned_show(struct gendisk *disk, char *page)
{
        if (blk_queue_is_zoned(disk->queue))
                return sprintf(page, "host-managed\n");
        return sprintf(page, "none\n");
}

static ssize_t queue_nr_zones_show(struct gendisk *disk, char *page)
{
        return queue_var_show(disk_nr_zones(disk), page);
}

static ssize_t queue_nomerges_show(struct gendisk *disk, char *page)
{
        return queue_var_show((blk_queue_nomerges(disk->queue) << 1) |
                               blk_queue_noxmerges(disk->queue), page);
}

static ssize_t queue_nomerges_store(struct gendisk *disk, const char *page,
                                    size_t count)
{
        unsigned long nm;
        ssize_t ret = queue_var_store(&nm, page, count);

        if (ret < 0)
                return ret;

        blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, disk->queue);
        blk_queue_flag_clear(QUEUE_FLAG_NOXMERGES, disk->queue);
        if (nm == 2)
                blk_queue_flag_set(QUEUE_FLAG_NOMERGES, disk->queue);
        else if (nm)
                blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, disk->queue);

        return ret;
}

static ssize_t queue_rq_affinity_show(struct gendisk *disk, char *page)
{
        bool set = test_bit(QUEUE_FLAG_SAME_COMP, &disk->queue->queue_flags);
        bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &disk->queue->queue_flags);

        return queue_var_show(set << force, page);
}

static ssize_t
queue_rq_affinity_store(struct gendisk *disk, const char *page, size_t count)
{
        ssize_t ret = -EINVAL;
#ifdef CONFIG_SMP
        struct request_queue *q = disk->queue;
        unsigned long val;

        ret = queue_var_store(&val, page, count);
        if (ret < 0)
                return ret;

        if (val == 2) {
                blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
                blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
        } else if (val == 1) {
                blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
                blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
        } else if (val == 0) {
                blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
                blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
        }
#endif
        return ret;
}

static ssize_t queue_poll_delay_store(struct gendisk *disk, const char *page,
                                size_t count)
{
        return count;
}

static ssize_t queue_poll_store(struct gendisk *disk, const char *page,
                                size_t count)
{
        if (!(disk->queue->limits.features & BLK_FEAT_POLL))
                return -EINVAL;
        pr_info_ratelimited("writes to the poll attribute are ignored.\n");
        pr_info_ratelimited("please use driver specific parameters instead.\n");
        return count;
}

static ssize_t queue_io_timeout_show(struct gendisk *disk, char *page)
{
        return sprintf(page, "%u\n", jiffies_to_msecs(disk->queue->rq_timeout));
}

static ssize_t queue_io_timeout_store(struct gendisk *disk, const char *page,
                                  size_t count)
{
        unsigned int val;
        int err;

        err = kstrtou32(page, 10, &val);
        if (err || val == 0)
                return -EINVAL;

        blk_queue_rq_timeout(disk->queue, msecs_to_jiffies(val));

        return count;
}

static ssize_t queue_wc_show(struct gendisk *disk, char *page)
{
        if (blk_queue_write_cache(disk->queue))
                return sprintf(page, "write back\n");
        return sprintf(page, "write through\n");
}

static ssize_t queue_wc_store(struct gendisk *disk, const char *page,
                              size_t count)
{
        struct queue_limits lim;
        bool disable;
        int err;

        if (!strncmp(page, "write back", 10)) {
                disable = false;
        } else if (!strncmp(page, "write through", 13) ||
                   !strncmp(page, "none", 4)) {
                disable = true;
        } else {
                return -EINVAL;
        }

        lim = queue_limits_start_update(disk->queue);
        if (disable)
                lim.flags |= BLK_FLAG_WRITE_CACHE_DISABLED;
        else
                lim.flags &= ~BLK_FLAG_WRITE_CACHE_DISABLED;
        err = queue_limits_commit_update(disk->queue, &lim);
        if (err)
                return err;
        return count;
}

#define QUEUE_RO_ENTRY(_prefix, _name)                        \
static struct queue_sysfs_entry _prefix##_entry = {        \
        .attr        = { .name = _name, .mode = 0444 },        \
        .show        = _prefix##_show,                        \
};

#define QUEUE_RW_ENTRY(_prefix, _name)                        \
static struct queue_sysfs_entry _prefix##_entry = {        \
        .attr        = { .name = _name, .mode = 0644 },        \
        .show        = _prefix##_show,                        \
        .store        = _prefix##_store,                        \
};

#define QUEUE_RW_LOAD_MODULE_ENTRY(_prefix, _name)                \
static struct queue_sysfs_entry _prefix##_entry = {                \
        .attr                = { .name = _name, .mode = 0644 },        \
        .show                = _prefix##_show,                        \
        .load_module        = _prefix##_load_module,                \
        .store                = _prefix##_store,                        \
}

QUEUE_RW_ENTRY(queue_requests, "nr_requests");
QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb");
QUEUE_RW_ENTRY(queue_max_sectors, "max_sectors_kb");
QUEUE_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb");
QUEUE_RO_ENTRY(queue_max_segments, "max_segments");
QUEUE_RO_ENTRY(queue_max_integrity_segments, "max_integrity_segments");
QUEUE_RO_ENTRY(queue_max_segment_size, "max_segment_size");
QUEUE_RW_LOAD_MODULE_ENTRY(elv_iosched, "scheduler");

QUEUE_RO_ENTRY(queue_logical_block_size, "logical_block_size");
QUEUE_RO_ENTRY(queue_physical_block_size, "physical_block_size");
QUEUE_RO_ENTRY(queue_chunk_sectors, "chunk_sectors");
QUEUE_RO_ENTRY(queue_io_min, "minimum_io_size");
QUEUE_RO_ENTRY(queue_io_opt, "optimal_io_size");

QUEUE_RO_ENTRY(queue_max_discard_segments, "max_discard_segments");
QUEUE_RO_ENTRY(queue_discard_granularity, "discard_granularity");
QUEUE_RO_ENTRY(queue_max_hw_discard_sectors, "discard_max_hw_bytes");
QUEUE_RW_ENTRY(queue_max_discard_sectors, "discard_max_bytes");
QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data");

QUEUE_RO_ENTRY(queue_atomic_write_max_sectors, "atomic_write_max_bytes");
QUEUE_RO_ENTRY(queue_atomic_write_boundary_sectors,
                "atomic_write_boundary_bytes");
QUEUE_RO_ENTRY(queue_atomic_write_unit_max, "atomic_write_unit_max_bytes");
QUEUE_RO_ENTRY(queue_atomic_write_unit_min, "atomic_write_unit_min_bytes");

QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");
QUEUE_RO_ENTRY(queue_max_write_zeroes_sectors, "write_zeroes_max_bytes");
QUEUE_RO_ENTRY(queue_zone_append_max, "zone_append_max_bytes");
QUEUE_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity");

QUEUE_RO_ENTRY(queue_zoned, "zoned");
QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
QUEUE_RO_ENTRY(queue_max_open_zones, "max_open_zones");
QUEUE_RO_ENTRY(queue_max_active_zones, "max_active_zones");

QUEUE_RW_ENTRY(queue_nomerges, "nomerges");
QUEUE_RW_ENTRY(queue_rq_affinity, "rq_affinity");
QUEUE_RW_ENTRY(queue_poll, "io_poll");
QUEUE_RW_ENTRY(queue_poll_delay, "io_poll_delay");
QUEUE_RW_ENTRY(queue_wc, "write_cache");
QUEUE_RO_ENTRY(queue_fua, "fua");
QUEUE_RO_ENTRY(queue_dax, "dax");
QUEUE_RW_ENTRY(queue_io_timeout, "io_timeout");
QUEUE_RO_ENTRY(queue_virt_boundary_mask, "virt_boundary_mask");
QUEUE_RO_ENTRY(queue_dma_alignment, "dma_alignment");

/* legacy alias for logical_block_size: */
static struct queue_sysfs_entry queue_hw_sector_size_entry = {
        .attr = {.name = "hw_sector_size", .mode = 0444 },
        .show = queue_logical_block_size_show,
};

QUEUE_RW_ENTRY(queue_rotational, "rotational");
QUEUE_RW_ENTRY(queue_iostats, "iostats");
QUEUE_RW_ENTRY(queue_add_random, "add_random");
QUEUE_RW_ENTRY(queue_stable_writes, "stable_writes");

#ifdef CONFIG_BLK_WBT
static ssize_t queue_var_store64(s64 *var, const char *page)
{
        int err;
        s64 v;

        err = kstrtos64(page, 10, &v);
        if (err < 0)
                return err;

        *var = v;
        return 0;
}

static ssize_t queue_wb_lat_show(struct gendisk *disk, char *page)
{
        if (!wbt_rq_qos(disk->queue))
                return -EINVAL;

        if (wbt_disabled(disk->queue))
                return sprintf(page, "0\n");

        return sprintf(page, "%llu\n",
                div_u64(wbt_get_min_lat(disk->queue), 1000));
}

static ssize_t queue_wb_lat_store(struct gendisk *disk, const char *page,
                                  size_t count)
{
        struct request_queue *q = disk->queue;
        struct rq_qos *rqos;
        ssize_t ret;
        s64 val;

        ret = queue_var_store64(&val, page);
        if (ret < 0)
                return ret;
        if (val < -1)
                return -EINVAL;

        rqos = wbt_rq_qos(q);
        if (!rqos) {
                ret = wbt_init(disk);
                if (ret)
                        return ret;
        }

        if (val == -1)
                val = wbt_default_latency_nsec(q);
        else if (val >= 0)
                val *= 1000ULL;

        if (wbt_get_min_lat(q) == val)
                return count;

        /*
         * Ensure that the queue is idled, in case the latency update
         * ends up either enabling or disabling wbt completely. We can't
         * have IO inflight if that happens.
         */
        blk_mq_quiesce_queue(q);

        wbt_set_min_lat(q, val);

        blk_mq_unquiesce_queue(q);

        return count;
}

QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");
#endif

/* Common attributes for bio-based and request-based queues. */
static struct attribute *queue_attrs[] = {
        &queue_ra_entry.attr,
        &queue_max_hw_sectors_entry.attr,
        &queue_max_sectors_entry.attr,
        &queue_max_segments_entry.attr,
        &queue_max_discard_segments_entry.attr,
        &queue_max_integrity_segments_entry.attr,
        &queue_max_segment_size_entry.attr,
        &queue_hw_sector_size_entry.attr,
        &queue_logical_block_size_entry.attr,
        &queue_physical_block_size_entry.attr,
        &queue_chunk_sectors_entry.attr,
        &queue_io_min_entry.attr,
        &queue_io_opt_entry.attr,
        &queue_discard_granularity_entry.attr,
        &queue_max_discard_sectors_entry.attr,
        &queue_max_hw_discard_sectors_entry.attr,
        &queue_discard_zeroes_data_entry.attr,
        &queue_atomic_write_max_sectors_entry.attr,
        &queue_atomic_write_boundary_sectors_entry.attr,
        &queue_atomic_write_unit_min_entry.attr,
        &queue_atomic_write_unit_max_entry.attr,
        &queue_write_same_max_entry.attr,
        &queue_max_write_zeroes_sectors_entry.attr,
        &queue_zone_append_max_entry.attr,
        &queue_zone_write_granularity_entry.attr,
        &queue_rotational_entry.attr,
        &queue_zoned_entry.attr,
        &queue_nr_zones_entry.attr,
        &queue_max_open_zones_entry.attr,
        &queue_max_active_zones_entry.attr,
        &queue_nomerges_entry.attr,
        &queue_iostats_entry.attr,
        &queue_stable_writes_entry.attr,
        &queue_add_random_entry.attr,
        &queue_poll_entry.attr,
        &queue_wc_entry.attr,
        &queue_fua_entry.attr,
        &queue_dax_entry.attr,
        &queue_poll_delay_entry.attr,
        &queue_virt_boundary_mask_entry.attr,
        &queue_dma_alignment_entry.attr,
        NULL,
};

/* Request-based queue attributes that are not relevant for bio-based queues. */
static struct attribute *blk_mq_queue_attrs[] = {
        &queue_requests_entry.attr,
        &elv_iosched_entry.attr,
        &queue_rq_affinity_entry.attr,
        &queue_io_timeout_entry.attr,
#ifdef CONFIG_BLK_WBT
        &queue_wb_lat_entry.attr,
#endif
        NULL,
};

static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr,
                                int n)
{
        struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
        struct request_queue *q = disk->queue;

        if ((attr == &queue_max_open_zones_entry.attr ||
             attr == &queue_max_active_zones_entry.attr) &&
            !blk_queue_is_zoned(q))
                return 0;

        return attr->mode;
}

static umode_t blk_mq_queue_attr_visible(struct kobject *kobj,
                                         struct attribute *attr, int n)
{
        struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
        struct request_queue *q = disk->queue;

        if (!queue_is_mq(q))
                return 0;

        if (attr == &queue_io_timeout_entry.attr && !q->mq_ops->timeout)
                return 0;

        return attr->mode;
}

static struct attribute_group queue_attr_group = {
        .attrs = queue_attrs,
        .is_visible = queue_attr_visible,
};

static struct attribute_group blk_mq_queue_attr_group = {
        .attrs = blk_mq_queue_attrs,
        .is_visible = blk_mq_queue_attr_visible,
};

#define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)

static ssize_t
queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
{
        struct queue_sysfs_entry *entry = to_queue(attr);
        struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
        ssize_t res;

        if (!entry->show)
                return -EIO;
        mutex_lock(&disk->queue->sysfs_lock);
        res = entry->show(disk, page);
        mutex_unlock(&disk->queue->sysfs_lock);
        return res;
}

static ssize_t
queue_attr_store(struct kobject *kobj, struct attribute *attr,
                    const char *page, size_t length)
{
        struct queue_sysfs_entry *entry = to_queue(attr);
        struct gendisk *disk = container_of(kobj, struct gendisk, queue_kobj);
        struct request_queue *q = disk->queue;
        ssize_t res;

        if (!entry->store)
                return -EIO;

        /*
         * If the attribute needs to load a module, do it before freezing the
         * queue to ensure that the module file can be read when the request
         * queue is the one for the device storing the module file.
         */
        if (entry->load_module) {
                res = entry->load_module(disk, page, length);
                if (res)
                        return res;
        }

        blk_mq_freeze_queue(q);
        mutex_lock(&q->sysfs_lock);
        res = entry->store(disk, page, length);
        mutex_unlock(&q->sysfs_lock);
        blk_mq_unfreeze_queue(q);
        return res;
}

static const struct sysfs_ops queue_sysfs_ops = {
        .show        = queue_attr_show,
        .store        = queue_attr_store,
};

static const struct attribute_group *blk_queue_attr_groups[] = {
        &queue_attr_group,
        &blk_mq_queue_attr_group,
        NULL
};

static void blk_queue_release(struct kobject *kobj)
{
        /* nothing to do here, all data is associated with the parent gendisk */
}

static const struct kobj_type blk_queue_ktype = {
        .default_groups = blk_queue_attr_groups,
        .sysfs_ops        = &queue_sysfs_ops,
        .release        = blk_queue_release,
};

static void blk_debugfs_remove(struct gendisk *disk)
{
        struct request_queue *q = disk->queue;

        mutex_lock(&q->debugfs_mutex);
        blk_trace_shutdown(q);
        debugfs_remove_recursive(q->debugfs_dir);
        q->debugfs_dir = NULL;
        q->sched_debugfs_dir = NULL;
        q->rqos_debugfs_dir = NULL;
        mutex_unlock(&q->debugfs_mutex);
}

/**
 * blk_register_queue - register a block layer queue with sysfs
 * @disk: Disk of which the request queue should be registered with sysfs.
 */
int blk_register_queue(struct gendisk *disk)
{
        struct request_queue *q = disk->queue;
        int ret;

        mutex_lock(&q->sysfs_dir_lock);
        kobject_init(&disk->queue_kobj, &blk_queue_ktype);
        ret = kobject_add(&disk->queue_kobj, &disk_to_dev(disk)->kobj, "queue");
        if (ret < 0)
                goto out_put_queue_kobj;

        if (queue_is_mq(q)) {
                ret = blk_mq_sysfs_register(disk);
                if (ret)
                        goto out_put_queue_kobj;
        }
        mutex_lock(&q->sysfs_lock);

        mutex_lock(&q->debugfs_mutex);
        q->debugfs_dir = debugfs_create_dir(disk->disk_name, blk_debugfs_root);
        if (queue_is_mq(q))
                blk_mq_debugfs_register(q);
        mutex_unlock(&q->debugfs_mutex);

        ret = disk_register_independent_access_ranges(disk);
        if (ret)
                goto out_debugfs_remove;

        if (q->elevator) {
                ret = elv_register_queue(q, false);
                if (ret)
                        goto out_unregister_ia_ranges;
        }

        ret = blk_crypto_sysfs_register(disk);
        if (ret)
                goto out_elv_unregister;

        blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
        wbt_enable_default(disk);

        /* Now everything is ready and send out KOBJ_ADD uevent */
        kobject_uevent(&disk->queue_kobj, KOBJ_ADD);
        if (q->elevator)
                kobject_uevent(&q->elevator->kobj, KOBJ_ADD);
        mutex_unlock(&q->sysfs_lock);
        mutex_unlock(&q->sysfs_dir_lock);

        /*
         * SCSI probing may synchronously create and destroy a lot of
         * request_queues for non-existent devices.  Shutting down a fully
         * functional queue takes measureable wallclock time as RCU grace
         * periods are involved.  To avoid excessive latency in these
         * cases, a request_queue starts out in a degraded mode which is
         * faster to shut down and is made fully functional here as
         * request_queues for non-existent devices never get registered.
         */
        if (!blk_queue_init_done(q)) {
                blk_queue_flag_set(QUEUE_FLAG_INIT_DONE, q);
                percpu_ref_switch_to_percpu(&q->q_usage_counter);
        }

        return ret;

out_elv_unregister:
        elv_unregister_queue(q);
out_unregister_ia_ranges:
        disk_unregister_independent_access_ranges(disk);
out_debugfs_remove:
        blk_debugfs_remove(disk);
        mutex_unlock(&q->sysfs_lock);
out_put_queue_kobj:
        kobject_put(&disk->queue_kobj);
        mutex_unlock(&q->sysfs_dir_lock);
        return ret;
}

/**
 * blk_unregister_queue - counterpart of blk_register_queue()
 * @disk: Disk of which the request queue should be unregistered from sysfs.
 *
 * Note: the caller is responsible for guaranteeing that this function is called
 * after blk_register_queue() has finished.
 */
void blk_unregister_queue(struct gendisk *disk)
{
        struct request_queue *q = disk->queue;

        if (WARN_ON(!q))
                return;

        /* Return early if disk->queue was never registered. */
        if (!blk_queue_registered(q))
                return;

        /*
         * Since sysfs_remove_dir() prevents adding new directory entries
         * before removal of existing entries starts, protect against
         * concurrent elv_iosched_store() calls.
         */
        mutex_lock(&q->sysfs_lock);
        blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
        mutex_unlock(&q->sysfs_lock);

        mutex_lock(&q->sysfs_dir_lock);
        /*
         * Remove the sysfs attributes before unregistering the queue data
         * structures that can be modified through sysfs.
         */
        if (queue_is_mq(q))
                blk_mq_sysfs_unregister(disk);
        blk_crypto_sysfs_unregister(disk);

        mutex_lock(&q->sysfs_lock);
        elv_unregister_queue(q);
        disk_unregister_independent_access_ranges(disk);
        mutex_unlock(&q->sysfs_lock);

        /* Now that we've deleted all child objects, we can delete the queue. */
        kobject_uevent(&disk->queue_kobj, KOBJ_REMOVE);
        kobject_del(&disk->queue_kobj);
        mutex_unlock(&q->sysfs_dir_lock);

        blk_debugfs_remove(disk);
}













































  410 






  497 









  500 
























 1215 








    6 















  836 

  627 
  463 






 1075 
 1000 
   73 















   34 








   38 


   38 











   43 


   17 




   34 









   35 



    3 


    3 


   36 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *  Copyright (C) 2008 Red Hat, Inc., Eric Paris <eparis@redhat.com>
 */

/*
 * Basic idea behind the notification queue: An fsnotify group (like inotify)
 * sends the userspace notification about events asynchronously some time after
 * the event happened.  When inotify gets an event it will need to add that
 * event to the group notify queue.  Since a single event might need to be on
 * multiple group's notification queues we can't add the event directly to each
 * queue and instead add a small "event_holder" to each queue.  This event_holder
 * has a pointer back to the original event.  Since the majority of events are
 * going to end up on one, and only one, notification queue we embed one
 * event_holder into each event.  This means we have a single allocation instead
 * of always needing two.  If the embedded event_holder is already in use by
 * another group a new event_holder (from fsnotify_event_holder_cachep) will be
 * allocated and used.
 */

#include <linux/fs.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/list.h>
#include <linux/module.h>
#include <linux/mount.h>
#include <linux/mutex.h>
#include <linux/namei.h>
#include <linux/path.h>
#include <linux/slab.h>
#include <linux/spinlock.h>

#include <linux/atomic.h>

#include <linux/fsnotify_backend.h>
#include "fsnotify.h"

static atomic_t fsnotify_sync_cookie = ATOMIC_INIT(0);

/**
 * fsnotify_get_cookie - return a unique cookie for use in synchronizing events.
 * Called from fsnotify_move, which is inlined into filesystem modules.
 */
u32 fsnotify_get_cookie(void)
{
        return atomic_inc_return(&fsnotify_sync_cookie);
}
EXPORT_SYMBOL_GPL(fsnotify_get_cookie);

void fsnotify_destroy_event(struct fsnotify_group *group,
                            struct fsnotify_event *event)
{
        /* Overflow events are per-group and we don't want to free them */
        if (!event || event == group->overflow_event)
                return;
        /*
         * If the event is still queued, we have a problem... Do an unreliable
         * lockless check first to avoid locking in the common case. The
         * locking may be necessary for permission events which got removed
         * from the list by a different CPU than the one freeing the event.
         */
        if (!list_empty(&event->list)) {
                spin_lock(&group->notification_lock);
                WARN_ON(!list_empty(&event->list));
                spin_unlock(&group->notification_lock);
        }
        group->ops->free_event(group, event);
}

/*
 * Try to add an event to the notification queue.
 * The group can later pull this event off the queue to deal with.
 * The group can use the @merge hook to merge the event with a queued event.
 * The group can use the @insert hook to insert the event into hash table.
 * The function returns:
 * 0 if the event was added to a queue
 * 1 if the event was merged with some other queued event
 * 2 if the event was not queued - either the queue of events has overflown
 *   or the group is shutting down.
 */
int fsnotify_insert_event(struct fsnotify_group *group,
                          struct fsnotify_event *event,
                          int (*merge)(struct fsnotify_group *,
                                       struct fsnotify_event *),
                          void (*insert)(struct fsnotify_group *,
                                         struct fsnotify_event *))
{
        int ret = 0;
        struct list_head *list = &group->notification_list;

        pr_debug("%s: group=%p event=%p\n", __func__, group, event);

        spin_lock(&group->notification_lock);

        if (group->shutdown) {
                spin_unlock(&group->notification_lock);
                return 2;
        }

        if (event == group->overflow_event ||
            group->q_len >= group->max_events) {
                ret = 2;
                /* Queue overflow event only if it isn't already queued */
                if (!list_empty(&group->overflow_event->list)) {
                        spin_unlock(&group->notification_lock);
                        return ret;
                }
                event = group->overflow_event;
                goto queue;
        }

        if (!list_empty(list) && merge) {
                ret = merge(group, event);
                if (ret) {
                        spin_unlock(&group->notification_lock);
                        return ret;
                }
        }

queue:
        group->q_len++;
        list_add_tail(&event->list, list);
        if (insert)
                insert(group, event);
        spin_unlock(&group->notification_lock);

        wake_up(&group->notification_waitq);
        kill_fasync(&group->fsn_fa, SIGIO, POLL_IN);
        return ret;
}

void fsnotify_remove_queued_event(struct fsnotify_group *group,
                                  struct fsnotify_event *event)
{
        assert_spin_locked(&group->notification_lock);
        /*
         * We need to init list head for the case of overflow event so that
         * check in fsnotify_add_event() works
         */
        list_del_init(&event->list);
        group->q_len--;
}

/*
 * Return the first event on the notification list without removing it.
 * Returns NULL if the list is empty.
 */
struct fsnotify_event *fsnotify_peek_first_event(struct fsnotify_group *group)
{
        assert_spin_locked(&group->notification_lock);

        if (fsnotify_notify_queue_is_empty(group))
                return NULL;

        return list_first_entry(&group->notification_list,
                                struct fsnotify_event, list);
}

/*
 * Remove and return the first event from the notification list.  It is the
 * responsibility of the caller to destroy the obtained event
 */
struct fsnotify_event *fsnotify_remove_first_event(struct fsnotify_group *group)
{
        struct fsnotify_event *event = fsnotify_peek_first_event(group);

        if (!event)
                return NULL;

        pr_debug("%s: group=%p event=%p\n", __func__, group, event);

        fsnotify_remove_queued_event(group, event);

        return event;
}

/*
 * Called when a group is being torn down to clean up any outstanding
 * event notifications.
 */
void fsnotify_flush_notify(struct fsnotify_group *group)
{
        struct fsnotify_event *event;

        spin_lock(&group->notification_lock);
        while (!fsnotify_notify_queue_is_empty(group)) {
                event = fsnotify_remove_first_event(group);
                spin_unlock(&group->notification_lock);
                fsnotify_destroy_event(group, event);
                spin_lock(&group->notification_lock);
        }
        spin_unlock(&group->notification_lock);
}







































   71 

















































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM vsock

#if !defined(_TRACE_VSOCK_VIRTIO_TRANSPORT_COMMON_H) || \
    defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_VSOCK_VIRTIO_TRANSPORT_COMMON_H

#include <linux/tracepoint.h>

TRACE_DEFINE_ENUM(VIRTIO_VSOCK_TYPE_STREAM);
TRACE_DEFINE_ENUM(VIRTIO_VSOCK_TYPE_SEQPACKET);

#define show_type(val) \
        __print_symbolic(val, \
                         { VIRTIO_VSOCK_TYPE_STREAM, "STREAM" }, \
                         { VIRTIO_VSOCK_TYPE_SEQPACKET, "SEQPACKET" })

TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_INVALID);
TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_REQUEST);
TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_RESPONSE);
TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_RST);
TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_SHUTDOWN);
TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_RW);
TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_CREDIT_UPDATE);
TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_CREDIT_REQUEST);

#define show_op(val) \
        __print_symbolic(val, \
                         { VIRTIO_VSOCK_OP_INVALID, "INVALID" }, \
                         { VIRTIO_VSOCK_OP_REQUEST, "REQUEST" }, \
                         { VIRTIO_VSOCK_OP_RESPONSE, "RESPONSE" }, \
                         { VIRTIO_VSOCK_OP_RST, "RST" }, \
                         { VIRTIO_VSOCK_OP_SHUTDOWN, "SHUTDOWN" }, \
                         { VIRTIO_VSOCK_OP_RW, "RW" }, \
                         { VIRTIO_VSOCK_OP_CREDIT_UPDATE, "CREDIT_UPDATE" }, \
                         { VIRTIO_VSOCK_OP_CREDIT_REQUEST, "CREDIT_REQUEST" })

TRACE_EVENT(virtio_transport_alloc_pkt,
        TP_PROTO(
                 __u32 src_cid, __u32 src_port,
                 __u32 dst_cid, __u32 dst_port,
                 __u32 len,
                 __u16 type,
                 __u16 op,
                 __u32 flags,
                 bool zcopy
        ),
        TP_ARGS(
                src_cid, src_port,
                dst_cid, dst_port,
                len,
                type,
                op,
                flags,
                zcopy
        ),
        TP_STRUCT__entry(
                __field(__u32, src_cid)
                __field(__u32, src_port)
                __field(__u32, dst_cid)
                __field(__u32, dst_port)
                __field(__u32, len)
                __field(__u16, type)
                __field(__u16, op)
                __field(__u32, flags)
                __field(bool, zcopy)
        ),
        TP_fast_assign(
                __entry->src_cid = src_cid;
                __entry->src_port = src_port;
                __entry->dst_cid = dst_cid;
                __entry->dst_port = dst_port;
                __entry->len = len;
                __entry->type = type;
                __entry->op = op;
                __entry->flags = flags;
                __entry->zcopy = zcopy;
        ),
        TP_printk("%u:%u -> %u:%u len=%u type=%s op=%s flags=%#x zcopy=%s",
                  __entry->src_cid, __entry->src_port,
                  __entry->dst_cid, __entry->dst_port,
                  __entry->len,
                  show_type(__entry->type),
                  show_op(__entry->op),
                  __entry->flags, __entry->zcopy ? "true" : "false")
);

TRACE_EVENT(virtio_transport_recv_pkt,
        TP_PROTO(
                 __u32 src_cid, __u32 src_port,
                 __u32 dst_cid, __u32 dst_port,
                 __u32 len,
                 __u16 type,
                 __u16 op,
                 __u32 flags,
                 __u32 buf_alloc,
                 __u32 fwd_cnt
        ),
        TP_ARGS(
                src_cid, src_port,
                dst_cid, dst_port,
                len,
                type,
                op,
                flags,
                buf_alloc,
                fwd_cnt
        ),
        TP_STRUCT__entry(
                __field(__u32, src_cid)
                __field(__u32, src_port)
                __field(__u32, dst_cid)
                __field(__u32, dst_port)
                __field(__u32, len)
                __field(__u16, type)
                __field(__u16, op)
                __field(__u32, flags)
                __field(__u32, buf_alloc)
                __field(__u32, fwd_cnt)
        ),
        TP_fast_assign(
                __entry->src_cid = src_cid;
                __entry->src_port = src_port;
                __entry->dst_cid = dst_cid;
                __entry->dst_port = dst_port;
                __entry->len = len;
                __entry->type = type;
                __entry->op = op;
                __entry->flags = flags;
                __entry->buf_alloc = buf_alloc;
                __entry->fwd_cnt = fwd_cnt;
        ),
        TP_printk("%u:%u -> %u:%u len=%u type=%s op=%s flags=%#x "
                  "buf_alloc=%u fwd_cnt=%u",
                  __entry->src_cid, __entry->src_port,
                  __entry->dst_cid, __entry->dst_port,
                  __entry->len,
                  show_type(__entry->type),
                  show_op(__entry->op),
                  __entry->flags,
                  __entry->buf_alloc,
                  __entry->fwd_cnt)
);

#endif /* _TRACE_VSOCK_VIRTIO_TRANSPORT_COMMON_H */

#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_FILE vsock_virtio_transport_common

/* This part must be outside protection */
#include <trace/define_trace.h>




































































































































































































































































   23 








   13 
   22 






   31 





























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
/*
 * linux/fs/nls/nls_cp866.c
 *
 * Charset cp866 translation tables.
 * Generated automatically from the Unicode and charset
 * tables from the Unicode Organization (www.unicode.org).
 * The Unicode to charset table has only exact mappings.
 */

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/nls.h>
#include <linux/errno.h>

static const wchar_t charset2uni[256] = {
        /* 0x00*/
        0x0000, 0x0001, 0x0002, 0x0003,
        0x0004, 0x0005, 0x0006, 0x0007,
        0x0008, 0x0009, 0x000a, 0x000b,
        0x000c, 0x000d, 0x000e, 0x000f,
        /* 0x10*/
        0x0010, 0x0011, 0x0012, 0x0013,
        0x0014, 0x0015, 0x0016, 0x0017,
        0x0018, 0x0019, 0x001a, 0x001b,
        0x001c, 0x001d, 0x001e, 0x001f,
        /* 0x20*/
        0x0020, 0x0021, 0x0022, 0x0023,
        0x0024, 0x0025, 0x0026, 0x0027,
        0x0028, 0x0029, 0x002a, 0x002b,
        0x002c, 0x002d, 0x002e, 0x002f,
        /* 0x30*/
        0x0030, 0x0031, 0x0032, 0x0033,
        0x0034, 0x0035, 0x0036, 0x0037,
        0x0038, 0x0039, 0x003a, 0x003b,
        0x003c, 0x003d, 0x003e, 0x003f,
        /* 0x40*/
        0x0040, 0x0041, 0x0042, 0x0043,
        0x0044, 0x0045, 0x0046, 0x0047,
        0x0048, 0x0049, 0x004a, 0x004b,
        0x004c, 0x004d, 0x004e, 0x004f,
        /* 0x50*/
        0x0050, 0x0051, 0x0052, 0x0053,
        0x0054, 0x0055, 0x0056, 0x0057,
        0x0058, 0x0059, 0x005a, 0x005b,
        0x005c, 0x005d, 0x005e, 0x005f,
        /* 0x60*/
        0x0060, 0x0061, 0x0062, 0x0063,
        0x0064, 0x0065, 0x0066, 0x0067,
        0x0068, 0x0069, 0x006a, 0x006b,
        0x006c, 0x006d, 0x006e, 0x006f,
        /* 0x70*/
        0x0070, 0x0071, 0x0072, 0x0073,
        0x0074, 0x0075, 0x0076, 0x0077,
        0x0078, 0x0079, 0x007a, 0x007b,
        0x007c, 0x007d, 0x007e, 0x007f,
        /* 0x80*/
        0x0410, 0x0411, 0x0412, 0x0413,
        0x0414, 0x0415, 0x0416, 0x0417,
        0x0418, 0x0419, 0x041a, 0x041b,
        0x041c, 0x041d, 0x041e, 0x041f,
        /* 0x90*/
        0x0420, 0x0421, 0x0422, 0x0423,
        0x0424, 0x0425, 0x0426, 0x0427,
        0x0428, 0x0429, 0x042a, 0x042b,
        0x042c, 0x042d, 0x042e, 0x042f,
        /* 0xa0*/
        0x0430, 0x0431, 0x0432, 0x0433,
        0x0434, 0x0435, 0x0436, 0x0437,
        0x0438, 0x0439, 0x043a, 0x043b,
        0x043c, 0x043d, 0x043e, 0x043f,
        /* 0xb0*/
        0x2591, 0x2592, 0x2593, 0x2502,
        0x2524, 0x2561, 0x2562, 0x2556,
        0x2555, 0x2563, 0x2551, 0x2557,
        0x255d, 0x255c, 0x255b, 0x2510,
        /* 0xc0*/
        0x2514, 0x2534, 0x252c, 0x251c,
        0x2500, 0x253c, 0x255e, 0x255f,
        0x255a, 0x2554, 0x2569, 0x2566,
        0x2560, 0x2550, 0x256c, 0x2567,
        /* 0xd0*/
        0x2568, 0x2564, 0x2565, 0x2559,
        0x2558, 0x2552, 0x2553, 0x256b,
        0x256a, 0x2518, 0x250c, 0x2588,
        0x2584, 0x258c, 0x2590, 0x2580,
        /* 0xe0*/
        0x0440, 0x0441, 0x0442, 0x0443,
        0x0444, 0x0445, 0x0446, 0x0447,
        0x0448, 0x0449, 0x044a, 0x044b,
        0x044c, 0x044d, 0x044e, 0x044f,
        /* 0xf0*/
        0x0401, 0x0451, 0x0404, 0x0454,
        0x0407, 0x0457, 0x040e, 0x045e,
        0x00b0, 0x2219, 0x00b7, 0x221a,
        0x2116, 0x00a4, 0x25a0, 0x00a0,
};

static const unsigned char page00[256] = {
        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */
        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */
        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */
        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */
        0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */
        0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */
        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */
        0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */
        0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */
        0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */
        0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */
        0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */
        0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */
        0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */
        0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */
        0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */

        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x80-0x87 */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x88-0x8f */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */
        0xff, 0x00, 0x00, 0x00, 0xfd, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa8-0xaf */
        0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfa, /* 0xb0-0xb7 */
};

static const unsigned char page04[256] = {
        0x00, 0xf0, 0x00, 0x00, 0xf2, 0x00, 0x00, 0xf4, /* 0x00-0x07 */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0x00, /* 0x08-0x0f */
        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x10-0x17 */
        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x18-0x1f */
        0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x20-0x27 */
        0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x28-0x2f */
        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0x30-0x37 */
        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0x38-0x3f */
        0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0x40-0x47 */
        0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0x48-0x4f */
        0x00, 0xf1, 0x00, 0x00, 0xf3, 0x00, 0x00, 0xf5, /* 0x50-0x57 */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf7, 0x00, /* 0x58-0x5f */
};

static const unsigned char page21[256] = {
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xfc, 0x00, /* 0x10-0x17 */
};

static const unsigned char page22[256] = {
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x08-0x0f */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x10-0x17 */
        0x00, 0xf9, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x18-0x1f */
};

static const unsigned char page25[256] = {
        0xc4, 0x00, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x00-0x07 */
        0x00, 0x00, 0x00, 0x00, 0xda, 0x00, 0x00, 0x00, /* 0x08-0x0f */
        0xbf, 0x00, 0x00, 0x00, 0xc0, 0x00, 0x00, 0x00, /* 0x10-0x17 */
        0xd9, 0x00, 0x00, 0x00, 0xc3, 0x00, 0x00, 0x00, /* 0x18-0x1f */
        0x00, 0x00, 0x00, 0x00, 0xb4, 0x00, 0x00, 0x00, /* 0x20-0x27 */
        0x00, 0x00, 0x00, 0x00, 0xc2, 0x00, 0x00, 0x00, /* 0x28-0x2f */
        0x00, 0x00, 0x00, 0x00, 0xc1, 0x00, 0x00, 0x00, /* 0x30-0x37 */
        0x00, 0x00, 0x00, 0x00, 0xc5, 0x00, 0x00, 0x00, /* 0x38-0x3f */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x40-0x47 */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x48-0x4f */
        0xcd, 0xba, 0xd5, 0xd6, 0xc9, 0xb8, 0xb7, 0xbb, /* 0x50-0x57 */
        0xd4, 0xd3, 0xc8, 0xbe, 0xbd, 0xbc, 0xc6, 0xc7, /* 0x58-0x5f */
        0xcc, 0xb5, 0xb6, 0xb9, 0xd1, 0xd2, 0xcb, 0xcf, /* 0x60-0x67 */
        0xd0, 0xca, 0xd8, 0xd7, 0xce, 0x00, 0x00, 0x00, /* 0x68-0x6f */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x70-0x77 */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x78-0x7f */

        0xdf, 0x00, 0x00, 0x00, 0xdc, 0x00, 0x00, 0x00, /* 0x80-0x87 */
        0xdb, 0x00, 0x00, 0x00, 0xdd, 0x00, 0x00, 0x00, /* 0x88-0x8f */
        0xde, 0xb0, 0xb1, 0xb2, 0x00, 0x00, 0x00, 0x00, /* 0x90-0x97 */
        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0x98-0x9f */
        0xfe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 0xa0-0xa7 */
};

static const unsigned char *const page_uni2charset[256] = {
        page00, NULL,   NULL,   NULL,   page04, NULL,   NULL,   NULL,   
        NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
        NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
        NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   NULL,   
        NULL,   page21, page22, NULL,   NULL,   page25, NULL,   NULL,   
};

static const unsigned char charset2lower[256] = {
        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */
        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */
        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */
        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */
        0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */
        0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */
        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */
        0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */
        0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x40-0x47 */
        0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x48-0x4f */
        0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x50-0x57 */
        0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */
        0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, /* 0x60-0x67 */
        0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, /* 0x68-0x6f */
        0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, /* 0x70-0x77 */
        0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */

        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0x80-0x87 */
        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0x88-0x8f */
        0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0x90-0x97 */
        0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0x98-0x9f */
        0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, /* 0xa0-0xa7 */
        0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, /* 0xa8-0xaf */
        0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */
        0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */
        0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */
        0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */
        0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */
        0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */
        0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, /* 0xe0-0xe7 */
        0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, /* 0xe8-0xef */
        0xf1, 0xf1, 0xf3, 0xf3, 0xf5, 0xf5, 0xf7, 0xf7, /* 0xf0-0xf7 */
        0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */
};

static const unsigned char charset2upper[256] = {
        0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, /* 0x00-0x07 */
        0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, /* 0x08-0x0f */
        0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, /* 0x10-0x17 */
        0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, /* 0x18-0x1f */
        0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, /* 0x20-0x27 */
        0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, /* 0x28-0x2f */
        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, /* 0x30-0x37 */
        0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, /* 0x38-0x3f */
        0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x40-0x47 */
        0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x48-0x4f */
        0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x50-0x57 */
        0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, /* 0x58-0x5f */
        0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, /* 0x60-0x67 */
        0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, /* 0x68-0x6f */
        0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, /* 0x70-0x77 */
        0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, /* 0x78-0x7f */

        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0x80-0x87 */
        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0x88-0x8f */
        0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0x90-0x97 */
        0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0x98-0x9f */
        0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, /* 0xa0-0xa7 */
        0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, /* 0xa8-0xaf */
        0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, /* 0xb0-0xb7 */
        0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, /* 0xb8-0xbf */
        0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, /* 0xc0-0xc7 */
        0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, /* 0xc8-0xcf */
        0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, /* 0xd0-0xd7 */
        0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, /* 0xd8-0xdf */
        0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, /* 0xe0-0xe7 */
        0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, /* 0xe8-0xef */
        0xf0, 0xf0, 0xf2, 0xf2, 0xf4, 0xf4, 0xf6, 0xf6, /* 0xf0-0xf7 */
        0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, /* 0xf8-0xff */
};

static int uni2char(wchar_t uni, unsigned char *out, int boundlen)
{
        const unsigned char *uni2charset;
        unsigned char cl = uni & 0x00ff;
        unsigned char ch = (uni & 0xff00) >> 8;

        if (boundlen <= 0)
                return -ENAMETOOLONG;

        uni2charset = page_uni2charset[ch];
        if (uni2charset && uni2charset[cl])
                out[0] = uni2charset[cl];
        else
                return -EINVAL;
        return 1;
}

static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni)
{
        *uni = charset2uni[*rawstring];
        if (*uni == 0x0000)
                return -EINVAL;
        return 1;
}

static struct nls_table table = {
        .charset        = "cp866",
        .uni2char        = uni2char,
        .char2uni        = char2uni,
        .charset2lower        = charset2lower,
        .charset2upper        = charset2upper,
};

static int __init init_nls_cp866(void)
{
        return register_nls(&table);
}

static void __exit exit_nls_cp866(void)
{
        unregister_nls(&table);
}

module_init(init_nls_cp866)
module_exit(exit_nls_cp866)

MODULE_DESCRIPTION("NLS Codepage 866 (Cyrillic/Russian)");
MODULE_LICENSE("Dual BSD/GPL");




















































    1 














    1 












    1 














































































































    1 










































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * TerraTec Cinergy T2/qanu USB2 DVB-T adapter.
 *
 * Copyright (C) 2007 Tomi Orava (tomimo@ncircle.nullnet.fi)
 *
 * Based on the dvb-usb-framework code and the
 * original Terratec Cinergy T2 driver by:
 *
 * Copyright (C) 2004 Daniel Mack <daniel@qanu.de> and
 *                    Holger Waechtler <holger@qanu.de>
 *
 *  Protocol Spec published on http://qanu.de/specs/terratec_cinergyT2.pdf
 */

#include "cinergyT2.h"


/* debug */
int dvb_usb_cinergyt2_debug;

module_param_named(debug, dvb_usb_cinergyt2_debug, int, 0644);
MODULE_PARM_DESC(debug, "set debugging level (1=info, xfer=2, rc=4 (or-able)).");

DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr);

struct cinergyt2_state {
        u8 rc_counter;
        unsigned char data[64];
};

/* Forward declaration */
static const struct dvb_usb_device_properties cinergyt2_properties;

static int cinergyt2_streaming_ctrl(struct dvb_usb_adapter *adap, int enable)
{
        struct dvb_usb_device *d = adap->dev;
        struct cinergyt2_state *st = d->priv;
        int ret;

        mutex_lock(&d->data_mutex);
        st->data[0] = CINERGYT2_EP1_CONTROL_STREAM_TRANSFER;
        st->data[1] = enable ? 1 : 0;

        ret = dvb_usb_generic_rw(d, st->data, 2, st->data, 64, 0);
        mutex_unlock(&d->data_mutex);

        return ret;
}

static int cinergyt2_power_ctrl(struct dvb_usb_device *d, int enable)
{
        struct cinergyt2_state *st = d->priv;
        int ret;

        mutex_lock(&d->data_mutex);
        st->data[0] = CINERGYT2_EP1_SLEEP_MODE;
        st->data[1] = enable ? 0 : 1;

        ret = dvb_usb_generic_rw(d, st->data, 2, st->data, 3, 0);
        mutex_unlock(&d->data_mutex);

        return ret;
}

static int cinergyt2_frontend_attach(struct dvb_usb_adapter *adap)
{
        struct dvb_usb_device *d = adap->dev;
        struct cinergyt2_state *st = d->priv;
        int ret;

        adap->fe_adap[0].fe = cinergyt2_fe_attach(adap->dev);

        mutex_lock(&d->data_mutex);
        st->data[0] = CINERGYT2_EP1_GET_FIRMWARE_VERSION;

        ret = dvb_usb_generic_rw(d, st->data, 1, st->data, 3, 0);
        if (ret < 0) {
                if (adap->fe_adap[0].fe)
                        adap->fe_adap[0].fe->ops.release(adap->fe_adap[0].fe);
                deb_rc("cinergyt2_power_ctrl() Failed to retrieve sleep state info\n");
        }
        mutex_unlock(&d->data_mutex);

        return ret;
}

static struct rc_map_table rc_map_cinergyt2_table[] = {
        { 0x0401, KEY_POWER },
        { 0x0402, KEY_1 },
        { 0x0403, KEY_2 },
        { 0x0404, KEY_3 },
        { 0x0405, KEY_4 },
        { 0x0406, KEY_5 },
        { 0x0407, KEY_6 },
        { 0x0408, KEY_7 },
        { 0x0409, KEY_8 },
        { 0x040a, KEY_9 },
        { 0x040c, KEY_0 },
        { 0x040b, KEY_VIDEO },
        { 0x040d, KEY_REFRESH },
        { 0x040e, KEY_SELECT },
        { 0x040f, KEY_EPG },
        { 0x0410, KEY_UP },
        { 0x0414, KEY_DOWN },
        { 0x0411, KEY_LEFT },
        { 0x0413, KEY_RIGHT },
        { 0x0412, KEY_OK },
        { 0x0415, KEY_TEXT },
        { 0x0416, KEY_INFO },
        { 0x0417, KEY_RED },
        { 0x0418, KEY_GREEN },
        { 0x0419, KEY_YELLOW },
        { 0x041a, KEY_BLUE },
        { 0x041c, KEY_VOLUMEUP },
        { 0x041e, KEY_VOLUMEDOWN },
        { 0x041d, KEY_MUTE },
        { 0x041b, KEY_CHANNELUP },
        { 0x041f, KEY_CHANNELDOWN },
        { 0x0440, KEY_PAUSE },
        { 0x044c, KEY_PLAY },
        { 0x0458, KEY_RECORD },
        { 0x0454, KEY_PREVIOUS },
        { 0x0448, KEY_STOP },
        { 0x045c, KEY_NEXT }
};

/* Number of keypresses to ignore before detect repeating */
#define RC_REPEAT_DELAY 3

static int repeatable_keys[] = {
        KEY_UP,
        KEY_DOWN,
        KEY_LEFT,
        KEY_RIGHT,
        KEY_VOLUMEUP,
        KEY_VOLUMEDOWN,
        KEY_CHANNELUP,
        KEY_CHANNELDOWN
};

static int cinergyt2_rc_query(struct dvb_usb_device *d, u32 *event, int *state)
{
        struct cinergyt2_state *st = d->priv;
        int i, ret;

        *state = REMOTE_NO_KEY_PRESSED;

        mutex_lock(&d->data_mutex);
        st->data[0] = CINERGYT2_EP1_GET_RC_EVENTS;

        ret = dvb_usb_generic_rw(d, st->data, 1, st->data, 5, 0);
        if (ret < 0)
                goto ret;

        if (st->data[4] == 0xff) {
                /* key repeat */
                st->rc_counter++;
                if (st->rc_counter > RC_REPEAT_DELAY) {
                        for (i = 0; i < ARRAY_SIZE(repeatable_keys); i++) {
                                if (d->last_event == repeatable_keys[i]) {
                                        *state = REMOTE_KEY_REPEAT;
                                        *event = d->last_event;
                                        deb_rc("repeat key, event %x\n",
                                                   *event);
                                        goto ret;
                                }
                        }
                        deb_rc("repeated key (non repeatable)\n");
                }
                goto ret;
        }

        /* hack to pass checksum on the custom field */
        st->data[2] = ~st->data[1];
        dvb_usb_nec_rc_key_to_event(d, st->data, event, state);
        if (st->data[0] != 0) {
                if (*event != d->last_event)
                        st->rc_counter = 0;

                deb_rc("key: %*ph\n", 5, st->data);
        }

ret:
        mutex_unlock(&d->data_mutex);
        return ret;
}

static int cinergyt2_usb_probe(struct usb_interface *intf,
                                const struct usb_device_id *id)
{
        return dvb_usb_device_init(intf, &cinergyt2_properties,
                                   THIS_MODULE, NULL, adapter_nr);
}

enum {
        TERRATEC_CINERGY_T2,
};

static struct usb_device_id cinergyt2_usb_table[] = {
        DVB_USB_DEV(TERRATEC, TERRATEC_CINERGY_T2),
        { }
};

MODULE_DEVICE_TABLE(usb, cinergyt2_usb_table);

static const struct dvb_usb_device_properties cinergyt2_properties = {
        .size_of_priv = sizeof(struct cinergyt2_state),
        .num_adapters = 1,
        .adapter = {
                {
                .num_frontends = 1,
                .fe = {{
                        .streaming_ctrl   = cinergyt2_streaming_ctrl,
                        .frontend_attach  = cinergyt2_frontend_attach,

                        /* parameter for the MPEG2-data transfer */
                        .stream = {
                                .type = USB_BULK,
                                .count = 5,
                                .endpoint = 0x02,
                                .u = {
                                        .bulk = {
                                                .buffersize = 512,
                                        }
                                }
                        },
                }},
                }
        },

        .power_ctrl       = cinergyt2_power_ctrl,

        .rc.legacy = {
                .rc_interval      = 50,
                .rc_map_table     = rc_map_cinergyt2_table,
                .rc_map_size      = ARRAY_SIZE(rc_map_cinergyt2_table),
                .rc_query         = cinergyt2_rc_query,
        },

        .generic_bulk_ctrl_endpoint = 1,

        .num_device_descs = 1,
        .devices = {
                { .name = "TerraTec/qanu USB2.0 Highspeed DVB-T Receiver",
                  .cold_ids = {NULL},
                  .warm_ids = { &cinergyt2_usb_table[TERRATEC_CINERGY_T2], NULL },
                },
                { NULL },
        }
};


static struct usb_driver cinergyt2_driver = {
        .name                = "cinergyT2",
        .probe                = cinergyt2_usb_probe,
        .disconnect        = dvb_usb_device_exit,
        .id_table        = cinergyt2_usb_table
};

module_usb_driver(cinergyt2_driver);

MODULE_DESCRIPTION("Terratec Cinergy T2 DVB-T driver");
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Tomi Orava");























































































































































































































































   18 
   18 





    5 



   11 











    6 





   12 
   11 

   11 

   12 


   11 









   12 



   25 
   25 






















    2 
































    6 





    3 

    3 



    6 

































































































































    2 






    1 












   12 




   12 
    2 
    2 















































































































































































   15 
   15 






























































































































































































































































































































































































    8 





















































































































































































































































































































































































































   16 







   16 




























































   14 





    3 




   12 



   15 
   15 

   15 



















































































































































































































































































































































































































   30 




    1 






    1 



   12 



    6 

    8 
    2 





   11 


    8 

    8 










   16 






   13 

    3 
































































































































































   10 




   10 


    8 
    7 
    3 

    8 


    8 







    8 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
// SPDX-License-Identifier: GPL-2.0
// Copyright (c) 2010-2011 EIA Electronics,
//                         Kurt Van Dijck <kurt.van.dijck@eia.be>
// Copyright (c) 2018 Protonic,
//                         Robin van der Gracht <robin@protonic.nl>
// Copyright (c) 2017-2019 Pengutronix,
//                         Marc Kleine-Budde <kernel@pengutronix.de>
// Copyright (c) 2017-2019 Pengutronix,
//                         Oleksij Rempel <kernel@pengutronix.de>

#include <linux/can/skb.h>

#include "j1939-priv.h"

#define J1939_XTP_TX_RETRY_LIMIT 100

#define J1939_ETP_PGN_CTL 0xc800
#define J1939_ETP_PGN_DAT 0xc700
#define J1939_TP_PGN_CTL 0xec00
#define J1939_TP_PGN_DAT 0xeb00

#define J1939_TP_CMD_RTS 0x10
#define J1939_TP_CMD_CTS 0x11
#define J1939_TP_CMD_EOMA 0x13
#define J1939_TP_CMD_BAM 0x20
#define J1939_TP_CMD_ABORT 0xff

#define J1939_ETP_CMD_RTS 0x14
#define J1939_ETP_CMD_CTS 0x15
#define J1939_ETP_CMD_DPO 0x16
#define J1939_ETP_CMD_EOMA 0x17
#define J1939_ETP_CMD_ABORT 0xff

enum j1939_xtp_abort {
        J1939_XTP_NO_ABORT = 0,
        J1939_XTP_ABORT_BUSY = 1,
        /* Already in one or more connection managed sessions and
         * cannot support another.
         *
         * EALREADY:
         * Operation already in progress
         */

        J1939_XTP_ABORT_RESOURCE = 2,
        /* System resources were needed for another task so this
         * connection managed session was terminated.
         *
         * EMSGSIZE:
         * The socket type requires that message be sent atomically,
         * and the size of the message to be sent made this
         * impossible.
         */

        J1939_XTP_ABORT_TIMEOUT = 3,
        /* A timeout occurred and this is the connection abort to
         * close the session.
         *
         * EHOSTUNREACH:
         * The destination host cannot be reached (probably because
         * the host is down or a remote router cannot reach it).
         */

        J1939_XTP_ABORT_GENERIC = 4,
        /* CTS messages received when data transfer is in progress
         *
         * EBADMSG:
         * Not a data message
         */

        J1939_XTP_ABORT_FAULT = 5,
        /* Maximal retransmit request limit reached
         *
         * ENOTRECOVERABLE:
         * State not recoverable
         */

        J1939_XTP_ABORT_UNEXPECTED_DATA = 6,
        /* Unexpected data transfer packet
         *
         * ENOTCONN:
         * Transport endpoint is not connected
         */

        J1939_XTP_ABORT_BAD_SEQ = 7,
        /* Bad sequence number (and software is not able to recover)
         *
         * EILSEQ:
         * Illegal byte sequence
         */

        J1939_XTP_ABORT_DUP_SEQ = 8,
        /* Duplicate sequence number (and software is not able to
         * recover)
         */

        J1939_XTP_ABORT_EDPO_UNEXPECTED = 9,
        /* Unexpected EDPO packet (ETP) or Message size > 1785 bytes
         * (TP)
         */

        J1939_XTP_ABORT_BAD_EDPO_PGN = 10,
        /* Unexpected EDPO PGN (PGN in EDPO is bad) */

        J1939_XTP_ABORT_EDPO_OUTOF_CTS = 11,
        /* EDPO number of packets is greater than CTS */

        J1939_XTP_ABORT_BAD_EDPO_OFFSET = 12,
        /* Bad EDPO offset */

        J1939_XTP_ABORT_OTHER_DEPRECATED = 13,
        /* Deprecated. Use 250 instead (Any other reason)  */

        J1939_XTP_ABORT_ECTS_UNXPECTED_PGN = 14,
        /* Unexpected ECTS PGN (PGN in ECTS is bad) */

        J1939_XTP_ABORT_ECTS_TOO_BIG = 15,
        /* ECTS requested packets exceeds message size */

        J1939_XTP_ABORT_OTHER = 250,
        /* Any other reason (if a Connection Abort reason is
         * identified that is not listed in the table use code 250)
         */
};

static unsigned int j1939_tp_block = 255;
static unsigned int j1939_tp_packet_delay;
static unsigned int j1939_tp_padding = 1;

/* helpers */
static const char *j1939_xtp_abort_to_str(enum j1939_xtp_abort abort)
{
        switch (abort) {
        case J1939_XTP_ABORT_BUSY:
                return "Already in one or more connection managed sessions and cannot support another.";
        case J1939_XTP_ABORT_RESOURCE:
                return "System resources were needed for another task so this connection managed session was terminated.";
        case J1939_XTP_ABORT_TIMEOUT:
                return "A timeout occurred and this is the connection abort to close the session.";
        case J1939_XTP_ABORT_GENERIC:
                return "CTS messages received when data transfer is in progress";
        case J1939_XTP_ABORT_FAULT:
                return "Maximal retransmit request limit reached";
        case J1939_XTP_ABORT_UNEXPECTED_DATA:
                return "Unexpected data transfer packet";
        case J1939_XTP_ABORT_BAD_SEQ:
                return "Bad sequence number (and software is not able to recover)";
        case J1939_XTP_ABORT_DUP_SEQ:
                return "Duplicate sequence number (and software is not able to recover)";
        case J1939_XTP_ABORT_EDPO_UNEXPECTED:
                return "Unexpected EDPO packet (ETP) or Message size > 1785 bytes (TP)";
        case J1939_XTP_ABORT_BAD_EDPO_PGN:
                return "Unexpected EDPO PGN (PGN in EDPO is bad)";
        case J1939_XTP_ABORT_EDPO_OUTOF_CTS:
                return "EDPO number of packets is greater than CTS";
        case J1939_XTP_ABORT_BAD_EDPO_OFFSET:
                return "Bad EDPO offset";
        case J1939_XTP_ABORT_OTHER_DEPRECATED:
                return "Deprecated. Use 250 instead (Any other reason)";
        case J1939_XTP_ABORT_ECTS_UNXPECTED_PGN:
                return "Unexpected ECTS PGN (PGN in ECTS is bad)";
        case J1939_XTP_ABORT_ECTS_TOO_BIG:
                return "ECTS requested packets exceeds message size";
        case J1939_XTP_ABORT_OTHER:
                return "Any other reason (if a Connection Abort reason is identified that is not listed in the table use code 250)";
        default:
                return "<unknown>";
        }
}

static int j1939_xtp_abort_to_errno(struct j1939_priv *priv,
                                    enum j1939_xtp_abort abort)
{
        int err;

        switch (abort) {
        case J1939_XTP_NO_ABORT:
                WARN_ON_ONCE(abort == J1939_XTP_NO_ABORT);
                err = 0;
                break;
        case J1939_XTP_ABORT_BUSY:
                err = EALREADY;
                break;
        case J1939_XTP_ABORT_RESOURCE:
                err = EMSGSIZE;
                break;
        case J1939_XTP_ABORT_TIMEOUT:
                err = EHOSTUNREACH;
                break;
        case J1939_XTP_ABORT_GENERIC:
                err = EBADMSG;
                break;
        case J1939_XTP_ABORT_FAULT:
                err = ENOTRECOVERABLE;
                break;
        case J1939_XTP_ABORT_UNEXPECTED_DATA:
                err = ENOTCONN;
                break;
        case J1939_XTP_ABORT_BAD_SEQ:
                err = EILSEQ;
                break;
        case J1939_XTP_ABORT_DUP_SEQ:
                err = EPROTO;
                break;
        case J1939_XTP_ABORT_EDPO_UNEXPECTED:
                err = EPROTO;
                break;
        case J1939_XTP_ABORT_BAD_EDPO_PGN:
                err = EPROTO;
                break;
        case J1939_XTP_ABORT_EDPO_OUTOF_CTS:
                err = EPROTO;
                break;
        case J1939_XTP_ABORT_BAD_EDPO_OFFSET:
                err = EPROTO;
                break;
        case J1939_XTP_ABORT_OTHER_DEPRECATED:
                err = EPROTO;
                break;
        case J1939_XTP_ABORT_ECTS_UNXPECTED_PGN:
                err = EPROTO;
                break;
        case J1939_XTP_ABORT_ECTS_TOO_BIG:
                err = EPROTO;
                break;
        case J1939_XTP_ABORT_OTHER:
                err = EPROTO;
                break;
        default:
                netdev_warn(priv->ndev, "Unknown abort code %i", abort);
                err = EPROTO;
        }

        return err;
}

static inline void j1939_session_list_lock(struct j1939_priv *priv)
{
        spin_lock_bh(&priv->active_session_list_lock);
}

static inline void j1939_session_list_unlock(struct j1939_priv *priv)
{
        spin_unlock_bh(&priv->active_session_list_lock);
}

void j1939_session_get(struct j1939_session *session)
{
        kref_get(&session->kref);
}

/* session completion functions */
static void __j1939_session_drop(struct j1939_session *session)
{
        if (!session->transmission)
                return;

        j1939_sock_pending_del(session->sk);
        sock_put(session->sk);
}

static void j1939_session_destroy(struct j1939_session *session)
{
        struct sk_buff *skb;

        if (session->transmission) {
                if (session->err)
                        j1939_sk_errqueue(session, J1939_ERRQUEUE_TX_ABORT);
                else
                        j1939_sk_errqueue(session, J1939_ERRQUEUE_TX_ACK);
        } else if (session->err) {
                        j1939_sk_errqueue(session, J1939_ERRQUEUE_RX_ABORT);
        }

        netdev_dbg(session->priv->ndev, "%s: 0x%p\n", __func__, session);

        WARN_ON_ONCE(!list_empty(&session->sk_session_queue_entry));
        WARN_ON_ONCE(!list_empty(&session->active_session_list_entry));

        while ((skb = skb_dequeue(&session->skb_queue)) != NULL) {
                /* drop ref taken in j1939_session_skb_queue() */
                skb_unref(skb);
                kfree_skb(skb);
        }
        __j1939_session_drop(session);
        j1939_priv_put(session->priv);
        kfree(session);
}

static void __j1939_session_release(struct kref *kref)
{
        struct j1939_session *session = container_of(kref, struct j1939_session,
                                                     kref);

        j1939_session_destroy(session);
}

void j1939_session_put(struct j1939_session *session)
{
        kref_put(&session->kref, __j1939_session_release);
}

static void j1939_session_txtimer_cancel(struct j1939_session *session)
{
        if (hrtimer_cancel(&session->txtimer))
                j1939_session_put(session);
}

static void j1939_session_rxtimer_cancel(struct j1939_session *session)
{
        if (hrtimer_cancel(&session->rxtimer))
                j1939_session_put(session);
}

void j1939_session_timers_cancel(struct j1939_session *session)
{
        j1939_session_txtimer_cancel(session);
        j1939_session_rxtimer_cancel(session);
}

static inline bool j1939_cb_is_broadcast(const struct j1939_sk_buff_cb *skcb)
{
        return (!skcb->addr.dst_name && (skcb->addr.da == 0xff));
}

static void j1939_session_skb_drop_old(struct j1939_session *session)
{
        struct sk_buff *do_skb;
        struct j1939_sk_buff_cb *do_skcb;
        unsigned int offset_start;
        unsigned long flags;

        if (skb_queue_len(&session->skb_queue) < 2)
                return;

        offset_start = session->pkt.tx_acked * 7;

        spin_lock_irqsave(&session->skb_queue.lock, flags);
        do_skb = skb_peek(&session->skb_queue);
        do_skcb = j1939_skb_to_cb(do_skb);

        if ((do_skcb->offset + do_skb->len) < offset_start) {
                __skb_unlink(do_skb, &session->skb_queue);
                /* drop ref taken in j1939_session_skb_queue() */
                skb_unref(do_skb);
                spin_unlock_irqrestore(&session->skb_queue.lock, flags);

                kfree_skb(do_skb);
        } else {
                spin_unlock_irqrestore(&session->skb_queue.lock, flags);
        }
}

void j1939_session_skb_queue(struct j1939_session *session,
                             struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct j1939_priv *priv = session->priv;

        j1939_ac_fixup(priv, skb);

        if (j1939_address_is_unicast(skcb->addr.da) &&
            priv->ents[skcb->addr.da].nusers)
                skcb->flags |= J1939_ECU_LOCAL_DST;

        skcb->flags |= J1939_ECU_LOCAL_SRC;

        skb_get(skb);
        skb_queue_tail(&session->skb_queue, skb);
}

static struct
sk_buff *j1939_session_skb_get_by_offset(struct j1939_session *session,
                                         unsigned int offset_start)
{
        struct j1939_priv *priv = session->priv;
        struct j1939_sk_buff_cb *do_skcb;
        struct sk_buff *skb = NULL;
        struct sk_buff *do_skb;
        unsigned long flags;

        spin_lock_irqsave(&session->skb_queue.lock, flags);
        skb_queue_walk(&session->skb_queue, do_skb) {
                do_skcb = j1939_skb_to_cb(do_skb);

                if (offset_start >= do_skcb->offset &&
                    offset_start < (do_skcb->offset + do_skb->len)) {
                        skb = do_skb;
                }
        }

        if (skb)
                skb_get(skb);

        spin_unlock_irqrestore(&session->skb_queue.lock, flags);

        if (!skb)
                netdev_dbg(priv->ndev, "%s: 0x%p: no skb found for start: %i, queue size: %i\n",
                           __func__, session, offset_start,
                           skb_queue_len(&session->skb_queue));

        return skb;
}

static struct sk_buff *j1939_session_skb_get(struct j1939_session *session)
{
        unsigned int offset_start;

        offset_start = session->pkt.dpo * 7;
        return j1939_session_skb_get_by_offset(session, offset_start);
}

/* see if we are receiver
 * returns 0 for broadcasts, although we will receive them
 */
static inline int j1939_tp_im_receiver(const struct j1939_sk_buff_cb *skcb)
{
        return skcb->flags & J1939_ECU_LOCAL_DST;
}

/* see if we are sender */
static inline int j1939_tp_im_transmitter(const struct j1939_sk_buff_cb *skcb)
{
        return skcb->flags & J1939_ECU_LOCAL_SRC;
}

/* see if we are involved as either receiver or transmitter */
static int j1939_tp_im_involved(const struct j1939_sk_buff_cb *skcb, bool swap)
{
        if (swap)
                return j1939_tp_im_receiver(skcb);
        else
                return j1939_tp_im_transmitter(skcb);
}

static int j1939_tp_im_involved_anydir(struct j1939_sk_buff_cb *skcb)
{
        return skcb->flags & (J1939_ECU_LOCAL_SRC | J1939_ECU_LOCAL_DST);
}

/* extract pgn from flow-ctl message */
static inline pgn_t j1939_xtp_ctl_to_pgn(const u8 *dat)
{
        pgn_t pgn;

        pgn = (dat[7] << 16) | (dat[6] << 8) | (dat[5] << 0);
        if (j1939_pgn_is_pdu1(pgn))
                pgn &= 0xffff00;
        return pgn;
}

static inline unsigned int j1939_tp_ctl_to_size(const u8 *dat)
{
        return (dat[2] << 8) + (dat[1] << 0);
}

static inline unsigned int j1939_etp_ctl_to_packet(const u8 *dat)
{
        return (dat[4] << 16) | (dat[3] << 8) | (dat[2] << 0);
}

static inline unsigned int j1939_etp_ctl_to_size(const u8 *dat)
{
        return (dat[4] << 24) | (dat[3] << 16) |
                (dat[2] << 8) | (dat[1] << 0);
}

/* find existing session:
 * reverse: swap cb's src & dst
 * there is no problem with matching broadcasts, since
 * broadcasts (no dst, no da) would never call this
 * with reverse == true
 */
static bool j1939_session_match(struct j1939_addr *se_addr,
                                struct j1939_addr *sk_addr, bool reverse)
{
        if (se_addr->type != sk_addr->type)
                return false;

        if (reverse) {
                if (se_addr->src_name) {
                        if (se_addr->src_name != sk_addr->dst_name)
                                return false;
                } else if (se_addr->sa != sk_addr->da) {
                        return false;
                }

                if (se_addr->dst_name) {
                        if (se_addr->dst_name != sk_addr->src_name)
                                return false;
                } else if (se_addr->da != sk_addr->sa) {
                        return false;
                }
        } else {
                if (se_addr->src_name) {
                        if (se_addr->src_name != sk_addr->src_name)
                                return false;
                } else if (se_addr->sa != sk_addr->sa) {
                        return false;
                }

                if (se_addr->dst_name) {
                        if (se_addr->dst_name != sk_addr->dst_name)
                                return false;
                } else if (se_addr->da != sk_addr->da) {
                        return false;
                }
        }

        return true;
}

static struct
j1939_session *j1939_session_get_by_addr_locked(struct j1939_priv *priv,
                                                struct list_head *root,
                                                struct j1939_addr *addr,
                                                bool reverse, bool transmitter)
{
        struct j1939_session *session;

        lockdep_assert_held(&priv->active_session_list_lock);

        list_for_each_entry(session, root, active_session_list_entry) {
                j1939_session_get(session);
                if (j1939_session_match(&session->skcb.addr, addr, reverse) &&
                    session->transmission == transmitter)
                        return session;
                j1939_session_put(session);
        }

        return NULL;
}

static struct
j1939_session *j1939_session_get_simple(struct j1939_priv *priv,
                                        struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct j1939_session *session;

        lockdep_assert_held(&priv->active_session_list_lock);

        list_for_each_entry(session, &priv->active_session_list,
                            active_session_list_entry) {
                j1939_session_get(session);
                if (session->skcb.addr.type == J1939_SIMPLE &&
                    session->tskey == skcb->tskey && session->sk == skb->sk)
                        return session;
                j1939_session_put(session);
        }

        return NULL;
}

static struct
j1939_session *j1939_session_get_by_addr(struct j1939_priv *priv,
                                         struct j1939_addr *addr,
                                         bool reverse, bool transmitter)
{
        struct j1939_session *session;

        j1939_session_list_lock(priv);
        session = j1939_session_get_by_addr_locked(priv,
                                                   &priv->active_session_list,
                                                   addr, reverse, transmitter);
        j1939_session_list_unlock(priv);

        return session;
}

static void j1939_skbcb_swap(struct j1939_sk_buff_cb *skcb)
{
        u8 tmp = 0;

        swap(skcb->addr.dst_name, skcb->addr.src_name);
        swap(skcb->addr.da, skcb->addr.sa);

        /* swap SRC and DST flags, leave other untouched */
        if (skcb->flags & J1939_ECU_LOCAL_SRC)
                tmp |= J1939_ECU_LOCAL_DST;
        if (skcb->flags & J1939_ECU_LOCAL_DST)
                tmp |= J1939_ECU_LOCAL_SRC;
        skcb->flags &= ~(J1939_ECU_LOCAL_SRC | J1939_ECU_LOCAL_DST);
        skcb->flags |= tmp;
}

static struct
sk_buff *j1939_tp_tx_dat_new(struct j1939_priv *priv,
                             const struct j1939_sk_buff_cb *re_skcb,
                             bool ctl,
                             bool swap_src_dst)
{
        struct sk_buff *skb;
        struct j1939_sk_buff_cb *skcb;

        skb = alloc_skb(sizeof(struct can_frame) + sizeof(struct can_skb_priv),
                        GFP_ATOMIC);
        if (unlikely(!skb))
                return ERR_PTR(-ENOMEM);

        skb->dev = priv->ndev;
        can_skb_reserve(skb);
        can_skb_prv(skb)->ifindex = priv->ndev->ifindex;
        can_skb_prv(skb)->skbcnt = 0;
        /* reserve CAN header */
        skb_reserve(skb, offsetof(struct can_frame, data));

        /* skb->cb must be large enough to hold a j1939_sk_buff_cb structure */
        BUILD_BUG_ON(sizeof(skb->cb) < sizeof(*re_skcb));

        memcpy(skb->cb, re_skcb, sizeof(*re_skcb));
        skcb = j1939_skb_to_cb(skb);
        if (swap_src_dst)
                j1939_skbcb_swap(skcb);

        if (ctl) {
                if (skcb->addr.type == J1939_ETP)
                        skcb->addr.pgn = J1939_ETP_PGN_CTL;
                else
                        skcb->addr.pgn = J1939_TP_PGN_CTL;
        } else {
                if (skcb->addr.type == J1939_ETP)
                        skcb->addr.pgn = J1939_ETP_PGN_DAT;
                else
                        skcb->addr.pgn = J1939_TP_PGN_DAT;
        }

        return skb;
}

/* TP transmit packet functions */
static int j1939_tp_tx_dat(struct j1939_session *session,
                           const u8 *dat, int len)
{
        struct j1939_priv *priv = session->priv;
        struct sk_buff *skb;

        skb = j1939_tp_tx_dat_new(priv, &session->skcb,
                                  false, false);
        if (IS_ERR(skb))
                return PTR_ERR(skb);

        skb_put_data(skb, dat, len);
        if (j1939_tp_padding && len < 8)
                memset(skb_put(skb, 8 - len), 0xff, 8 - len);

        return j1939_send_one(priv, skb);
}

static int j1939_xtp_do_tx_ctl(struct j1939_priv *priv,
                               const struct j1939_sk_buff_cb *re_skcb,
                               bool swap_src_dst, pgn_t pgn, const u8 *dat)
{
        struct sk_buff *skb;
        u8 *skdat;

        if (!j1939_tp_im_involved(re_skcb, swap_src_dst))
                return 0;

        skb = j1939_tp_tx_dat_new(priv, re_skcb, true, swap_src_dst);
        if (IS_ERR(skb))
                return PTR_ERR(skb);

        skdat = skb_put(skb, 8);
        memcpy(skdat, dat, 5);
        skdat[5] = (pgn >> 0);
        skdat[6] = (pgn >> 8);
        skdat[7] = (pgn >> 16);

        return j1939_send_one(priv, skb);
}

static inline int j1939_tp_tx_ctl(struct j1939_session *session,
                                  bool swap_src_dst, const u8 *dat)
{
        struct j1939_priv *priv = session->priv;

        return j1939_xtp_do_tx_ctl(priv, &session->skcb,
                                   swap_src_dst,
                                   session->skcb.addr.pgn, dat);
}

static int j1939_xtp_tx_abort(struct j1939_priv *priv,
                              const struct j1939_sk_buff_cb *re_skcb,
                              bool swap_src_dst,
                              enum j1939_xtp_abort err,
                              pgn_t pgn)
{
        u8 dat[5];

        if (!j1939_tp_im_involved(re_skcb, swap_src_dst))
                return 0;

        memset(dat, 0xff, sizeof(dat));
        dat[0] = J1939_TP_CMD_ABORT;
        dat[1] = err;
        return j1939_xtp_do_tx_ctl(priv, re_skcb, swap_src_dst, pgn, dat);
}

void j1939_tp_schedule_txtimer(struct j1939_session *session, int msec)
{
        j1939_session_get(session);
        hrtimer_start(&session->txtimer, ms_to_ktime(msec),
                      HRTIMER_MODE_REL_SOFT);
}

static inline void j1939_tp_set_rxtimeout(struct j1939_session *session,
                                          int msec)
{
        j1939_session_rxtimer_cancel(session);
        j1939_session_get(session);
        hrtimer_start(&session->rxtimer, ms_to_ktime(msec),
                      HRTIMER_MODE_REL_SOFT);
}

static int j1939_session_tx_rts(struct j1939_session *session)
{
        u8 dat[8];
        int ret;

        memset(dat, 0xff, sizeof(dat));

        dat[1] = (session->total_message_size >> 0);
        dat[2] = (session->total_message_size >> 8);
        dat[3] = session->pkt.total;

        if (session->skcb.addr.type == J1939_ETP) {
                dat[0] = J1939_ETP_CMD_RTS;
                dat[1] = (session->total_message_size >> 0);
                dat[2] = (session->total_message_size >> 8);
                dat[3] = (session->total_message_size >> 16);
                dat[4] = (session->total_message_size >> 24);
        } else if (j1939_cb_is_broadcast(&session->skcb)) {
                dat[0] = J1939_TP_CMD_BAM;
                /* fake cts for broadcast */
                session->pkt.tx = 0;
        } else {
                dat[0] = J1939_TP_CMD_RTS;
                dat[4] = dat[3];
        }

        if (dat[0] == session->last_txcmd)
                /* done already */
                return 0;

        ret = j1939_tp_tx_ctl(session, false, dat);
        if (ret < 0)
                return ret;

        session->last_txcmd = dat[0];
        if (dat[0] == J1939_TP_CMD_BAM) {
                j1939_tp_schedule_txtimer(session, 50);
                j1939_tp_set_rxtimeout(session, 250);
        } else {
                j1939_tp_set_rxtimeout(session, 1250);
        }

        netdev_dbg(session->priv->ndev, "%s: 0x%p\n", __func__, session);

        return 0;
}

static int j1939_session_tx_dpo(struct j1939_session *session)
{
        unsigned int pkt;
        u8 dat[8];
        int ret;

        memset(dat, 0xff, sizeof(dat));

        dat[0] = J1939_ETP_CMD_DPO;
        session->pkt.dpo = session->pkt.tx_acked;
        pkt = session->pkt.dpo;
        dat[1] = session->pkt.last - session->pkt.tx_acked;
        dat[2] = (pkt >> 0);
        dat[3] = (pkt >> 8);
        dat[4] = (pkt >> 16);

        ret = j1939_tp_tx_ctl(session, false, dat);
        if (ret < 0)
                return ret;

        session->last_txcmd = dat[0];
        j1939_tp_set_rxtimeout(session, 1250);
        session->pkt.tx = session->pkt.tx_acked;

        netdev_dbg(session->priv->ndev, "%s: 0x%p\n", __func__, session);

        return 0;
}

static int j1939_session_tx_dat(struct j1939_session *session)
{
        struct j1939_priv *priv = session->priv;
        struct j1939_sk_buff_cb *se_skcb;
        int offset, pkt_done, pkt_end;
        unsigned int len, pdelay;
        struct sk_buff *se_skb;
        const u8 *tpdat;
        int ret = 0;
        u8 dat[8];

        se_skb = j1939_session_skb_get_by_offset(session, session->pkt.tx * 7);
        if (!se_skb)
                return -ENOBUFS;

        se_skcb = j1939_skb_to_cb(se_skb);
        tpdat = se_skb->data;
        ret = 0;
        pkt_done = 0;
        if (session->skcb.addr.type != J1939_ETP &&
            j1939_cb_is_broadcast(&session->skcb))
                pkt_end = session->pkt.total;
        else
                pkt_end = session->pkt.last;

        while (session->pkt.tx < pkt_end) {
                dat[0] = session->pkt.tx - session->pkt.dpo + 1;
                offset = (session->pkt.tx * 7) - se_skcb->offset;
                len =  se_skb->len - offset;
                if (len > 7)
                        len = 7;

                if (offset + len > se_skb->len) {
                        netdev_err_once(priv->ndev,
                                        "%s: 0x%p: requested data outside of queued buffer: offset %i, len %i, pkt.tx: %i\n",
                                        __func__, session, se_skcb->offset,
                                        se_skb->len , session->pkt.tx);
                        ret = -EOVERFLOW;
                        goto out_free;
                }

                if (!len) {
                        ret = -ENOBUFS;
                        break;
                }

                memcpy(&dat[1], &tpdat[offset], len);
                ret = j1939_tp_tx_dat(session, dat, len + 1);
                if (ret < 0) {
                        /* ENOBUFS == CAN interface TX queue is full */
                        if (ret != -ENOBUFS)
                                netdev_alert(priv->ndev,
                                             "%s: 0x%p: queue data error: %i\n",
                                             __func__, session, ret);
                        break;
                }

                session->last_txcmd = 0xff;
                pkt_done++;
                session->pkt.tx++;
                pdelay = j1939_cb_is_broadcast(&session->skcb) ? 50 :
                        j1939_tp_packet_delay;

                if (session->pkt.tx < session->pkt.total && pdelay) {
                        j1939_tp_schedule_txtimer(session, pdelay);
                        break;
                }
        }

        if (pkt_done)
                j1939_tp_set_rxtimeout(session, 250);

 out_free:
        if (ret)
                kfree_skb(se_skb);
        else
                consume_skb(se_skb);

        return ret;
}

static int j1939_xtp_txnext_transmiter(struct j1939_session *session)
{
        struct j1939_priv *priv = session->priv;
        int ret = 0;

        if (!j1939_tp_im_transmitter(&session->skcb)) {
                netdev_alert(priv->ndev, "%s: 0x%p: called by not transmitter!\n",
                             __func__, session);
                return -EINVAL;
        }

        switch (session->last_cmd) {
        case 0:
                ret = j1939_session_tx_rts(session);
                break;

        case J1939_ETP_CMD_CTS:
                if (session->last_txcmd != J1939_ETP_CMD_DPO) {
                        ret = j1939_session_tx_dpo(session);
                        if (ret)
                                return ret;
                }

                fallthrough;
        case J1939_TP_CMD_CTS:
        case 0xff: /* did some data */
        case J1939_ETP_CMD_DPO:
        case J1939_TP_CMD_BAM:
                ret = j1939_session_tx_dat(session);

                break;
        default:
                netdev_alert(priv->ndev, "%s: 0x%p: unexpected last_cmd: %x\n",
                             __func__, session, session->last_cmd);
        }

        return ret;
}

static int j1939_session_tx_cts(struct j1939_session *session)
{
        struct j1939_priv *priv = session->priv;
        unsigned int pkt, len;
        int ret;
        u8 dat[8];

        if (!j1939_sk_recv_match(priv, &session->skcb))
                return -ENOENT;

        len = session->pkt.total - session->pkt.rx;
        len = min3(len, session->pkt.block, j1939_tp_block ?: 255);
        memset(dat, 0xff, sizeof(dat));

        if (session->skcb.addr.type == J1939_ETP) {
                pkt = session->pkt.rx + 1;
                dat[0] = J1939_ETP_CMD_CTS;
                dat[1] = len;
                dat[2] = (pkt >> 0);
                dat[3] = (pkt >> 8);
                dat[4] = (pkt >> 16);
        } else {
                dat[0] = J1939_TP_CMD_CTS;
                dat[1] = len;
                dat[2] = session->pkt.rx + 1;
        }

        if (dat[0] == session->last_txcmd)
                /* done already */
                return 0;

        ret = j1939_tp_tx_ctl(session, true, dat);
        if (ret < 0)
                return ret;

        if (len)
                /* only mark cts done when len is set */
                session->last_txcmd = dat[0];
        j1939_tp_set_rxtimeout(session, 1250);

        netdev_dbg(session->priv->ndev, "%s: 0x%p\n", __func__, session);

        return 0;
}

static int j1939_session_tx_eoma(struct j1939_session *session)
{
        struct j1939_priv *priv = session->priv;
        u8 dat[8];
        int ret;

        if (!j1939_sk_recv_match(priv, &session->skcb))
                return -ENOENT;

        memset(dat, 0xff, sizeof(dat));

        if (session->skcb.addr.type == J1939_ETP) {
                dat[0] = J1939_ETP_CMD_EOMA;
                dat[1] = session->total_message_size >> 0;
                dat[2] = session->total_message_size >> 8;
                dat[3] = session->total_message_size >> 16;
                dat[4] = session->total_message_size >> 24;
        } else {
                dat[0] = J1939_TP_CMD_EOMA;
                dat[1] = session->total_message_size;
                dat[2] = session->total_message_size >> 8;
                dat[3] = session->pkt.total;
        }

        if (dat[0] == session->last_txcmd)
                /* done already */
                return 0;

        ret = j1939_tp_tx_ctl(session, true, dat);
        if (ret < 0)
                return ret;

        session->last_txcmd = dat[0];

        /* wait for the EOMA packet to come in */
        j1939_tp_set_rxtimeout(session, 1250);

        netdev_dbg(session->priv->ndev, "%s: 0x%p\n", __func__, session);

        return 0;
}

static int j1939_xtp_txnext_receiver(struct j1939_session *session)
{
        struct j1939_priv *priv = session->priv;
        int ret = 0;

        if (!j1939_tp_im_receiver(&session->skcb)) {
                netdev_alert(priv->ndev, "%s: 0x%p: called by not receiver!\n",
                             __func__, session);
                return -EINVAL;
        }

        switch (session->last_cmd) {
        case J1939_TP_CMD_RTS:
        case J1939_ETP_CMD_RTS:
                ret = j1939_session_tx_cts(session);
                break;

        case J1939_ETP_CMD_CTS:
        case J1939_TP_CMD_CTS:
        case 0xff: /* did some data */
        case J1939_ETP_CMD_DPO:
                if ((session->skcb.addr.type == J1939_TP &&
                     j1939_cb_is_broadcast(&session->skcb)))
                        break;

                if (session->pkt.rx >= session->pkt.total) {
                        ret = j1939_session_tx_eoma(session);
                } else if (session->pkt.rx >= session->pkt.last) {
                        session->last_txcmd = 0;
                        ret = j1939_session_tx_cts(session);
                }
                break;
        default:
                netdev_alert(priv->ndev, "%s: 0x%p: unexpected last_cmd: %x\n",
                             __func__, session, session->last_cmd);
        }

        return ret;
}

static int j1939_simple_txnext(struct j1939_session *session)
{
        struct j1939_priv *priv = session->priv;
        struct sk_buff *se_skb = j1939_session_skb_get(session);
        struct sk_buff *skb;
        int ret;

        if (!se_skb)
                return 0;

        skb = skb_clone(se_skb, GFP_ATOMIC);
        if (!skb) {
                ret = -ENOMEM;
                goto out_free;
        }

        can_skb_set_owner(skb, se_skb->sk);

        j1939_tp_set_rxtimeout(session, J1939_SIMPLE_ECHO_TIMEOUT_MS);

        ret = j1939_send_one(priv, skb);
        if (ret)
                goto out_free;

        j1939_sk_errqueue(session, J1939_ERRQUEUE_TX_SCHED);
        j1939_sk_queue_activate_next(session);

 out_free:
        if (ret)
                kfree_skb(se_skb);
        else
                consume_skb(se_skb);

        return ret;
}

static bool j1939_session_deactivate_locked(struct j1939_session *session)
{
        bool active = false;

        lockdep_assert_held(&session->priv->active_session_list_lock);

        if (session->state >= J1939_SESSION_ACTIVE &&
            session->state < J1939_SESSION_ACTIVE_MAX) {
                active = true;

                list_del_init(&session->active_session_list_entry);
                session->state = J1939_SESSION_DONE;
                j1939_session_put(session);
        }

        return active;
}

static bool j1939_session_deactivate(struct j1939_session *session)
{
        struct j1939_priv *priv = session->priv;
        bool active;

        j1939_session_list_lock(priv);
        active = j1939_session_deactivate_locked(session);
        j1939_session_list_unlock(priv);

        return active;
}

static void
j1939_session_deactivate_activate_next(struct j1939_session *session)
{
        if (j1939_session_deactivate(session))
                j1939_sk_queue_activate_next(session);
}

static void __j1939_session_cancel(struct j1939_session *session,
                                   enum j1939_xtp_abort err)
{
        struct j1939_priv *priv = session->priv;

        WARN_ON_ONCE(!err);
        lockdep_assert_held(&session->priv->active_session_list_lock);

        session->err = j1939_xtp_abort_to_errno(priv, err);
        session->state = J1939_SESSION_WAITING_ABORT;
        /* do not send aborts on incoming broadcasts */
        if (!j1939_cb_is_broadcast(&session->skcb)) {
                j1939_xtp_tx_abort(priv, &session->skcb,
                                   !session->transmission,
                                   err, session->skcb.addr.pgn);
        }

        if (session->sk)
                j1939_sk_send_loop_abort(session->sk, session->err);
}

static void j1939_session_cancel(struct j1939_session *session,
                                 enum j1939_xtp_abort err)
{
        j1939_session_list_lock(session->priv);

        if (session->state >= J1939_SESSION_ACTIVE &&
            session->state < J1939_SESSION_WAITING_ABORT) {
                j1939_tp_set_rxtimeout(session, J1939_XTP_ABORT_TIMEOUT_MS);
                __j1939_session_cancel(session, err);
        }

        j1939_session_list_unlock(session->priv);

        if (!session->sk)
                j1939_sk_errqueue(session, J1939_ERRQUEUE_RX_ABORT);
}

static enum hrtimer_restart j1939_tp_txtimer(struct hrtimer *hrtimer)
{
        struct j1939_session *session =
                container_of(hrtimer, struct j1939_session, txtimer);
        struct j1939_priv *priv = session->priv;
        int ret = 0;

        if (session->skcb.addr.type == J1939_SIMPLE) {
                ret = j1939_simple_txnext(session);
        } else {
                if (session->transmission)
                        ret = j1939_xtp_txnext_transmiter(session);
                else
                        ret = j1939_xtp_txnext_receiver(session);
        }

        switch (ret) {
        case -ENOBUFS:
                /* Retry limit is currently arbitrary chosen */
                if (session->tx_retry < J1939_XTP_TX_RETRY_LIMIT) {
                        session->tx_retry++;
                        j1939_tp_schedule_txtimer(session,
                                                  10 + get_random_u32_below(16));
                } else {
                        netdev_alert(priv->ndev, "%s: 0x%p: tx retry count reached\n",
                                     __func__, session);
                        session->err = -ENETUNREACH;
                        j1939_session_rxtimer_cancel(session);
                        j1939_session_deactivate_activate_next(session);
                }
                break;
        case -ENETDOWN:
                /* In this case we should get a netdev_event(), all active
                 * sessions will be cleared by j1939_cancel_active_session().
                 * So handle this as an error, but let
                 * j1939_cancel_active_session() do the cleanup including
                 * propagation of the error to user space.
                 */
                break;
        case -EOVERFLOW:
                j1939_session_cancel(session, J1939_XTP_ABORT_ECTS_TOO_BIG);
                break;
        case 0:
                session->tx_retry = 0;
                break;
        default:
                netdev_alert(priv->ndev, "%s: 0x%p: tx aborted with unknown reason: %i\n",
                             __func__, session, ret);
                if (session->skcb.addr.type != J1939_SIMPLE) {
                        j1939_session_cancel(session, J1939_XTP_ABORT_OTHER);
                } else {
                        session->err = ret;
                        j1939_session_rxtimer_cancel(session);
                        j1939_session_deactivate_activate_next(session);
                }
        }

        j1939_session_put(session);

        return HRTIMER_NORESTART;
}

static void j1939_session_completed(struct j1939_session *session)
{
        struct sk_buff *se_skb;

        if (!session->transmission) {
                se_skb = j1939_session_skb_get(session);
                /* distribute among j1939 receivers */
                j1939_sk_recv(session->priv, se_skb);
                consume_skb(se_skb);
        }

        j1939_session_deactivate_activate_next(session);
}

static enum hrtimer_restart j1939_tp_rxtimer(struct hrtimer *hrtimer)
{
        struct j1939_session *session = container_of(hrtimer,
                                                     struct j1939_session,
                                                     rxtimer);
        struct j1939_priv *priv = session->priv;

        if (session->state == J1939_SESSION_WAITING_ABORT) {
                netdev_alert(priv->ndev, "%s: 0x%p: abort rx timeout. Force session deactivation\n",
                             __func__, session);

                j1939_session_deactivate_activate_next(session);

        } else if (session->skcb.addr.type == J1939_SIMPLE) {
                netdev_alert(priv->ndev, "%s: 0x%p: Timeout. Failed to send simple message.\n",
                             __func__, session);

                /* The message is probably stuck in the CAN controller and can
                 * be send as soon as CAN bus is in working state again.
                 */
                session->err = -ETIME;
                j1939_session_deactivate(session);
        } else {
                j1939_session_list_lock(session->priv);
                if (session->state >= J1939_SESSION_ACTIVE &&
                    session->state < J1939_SESSION_ACTIVE_MAX) {
                        netdev_alert(priv->ndev, "%s: 0x%p: rx timeout, send abort\n",
                                     __func__, session);
                        j1939_session_get(session);
                        hrtimer_start(&session->rxtimer,
                                      ms_to_ktime(J1939_XTP_ABORT_TIMEOUT_MS),
                                      HRTIMER_MODE_REL_SOFT);
                        __j1939_session_cancel(session, J1939_XTP_ABORT_TIMEOUT);
                }
                j1939_session_list_unlock(session->priv);

                if (!session->sk)
                        j1939_sk_errqueue(session, J1939_ERRQUEUE_RX_ABORT);
        }

        j1939_session_put(session);

        return HRTIMER_NORESTART;
}

static bool j1939_xtp_rx_cmd_bad_pgn(struct j1939_session *session,
                                     const struct sk_buff *skb)
{
        const struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        pgn_t pgn = j1939_xtp_ctl_to_pgn(skb->data);
        struct j1939_priv *priv = session->priv;
        enum j1939_xtp_abort abort = J1939_XTP_NO_ABORT;
        u8 cmd = skb->data[0];

        if (session->skcb.addr.pgn == pgn)
                return false;

        switch (cmd) {
        case J1939_TP_CMD_BAM:
                abort = J1939_XTP_NO_ABORT;
                break;

        case J1939_ETP_CMD_RTS:
                fallthrough;
        case J1939_TP_CMD_RTS:
                abort = J1939_XTP_ABORT_BUSY;
                break;

        case J1939_ETP_CMD_CTS:
                fallthrough;
        case J1939_TP_CMD_CTS:
                abort = J1939_XTP_ABORT_ECTS_UNXPECTED_PGN;
                break;

        case J1939_ETP_CMD_DPO:
                abort = J1939_XTP_ABORT_BAD_EDPO_PGN;
                break;

        case J1939_ETP_CMD_EOMA:
                fallthrough;
        case J1939_TP_CMD_EOMA:
                abort = J1939_XTP_ABORT_OTHER;
                break;

        case J1939_ETP_CMD_ABORT: /* && J1939_TP_CMD_ABORT */
                abort = J1939_XTP_NO_ABORT;
                break;

        default:
                WARN_ON_ONCE(1);
                break;
        }

        netdev_warn(priv->ndev, "%s: 0x%p: CMD 0x%02x with PGN 0x%05x for running session with different PGN 0x%05x.\n",
                    __func__, session, cmd, pgn, session->skcb.addr.pgn);
        if (abort != J1939_XTP_NO_ABORT)
                j1939_xtp_tx_abort(priv, skcb, true, abort, pgn);

        return true;
}

static void j1939_xtp_rx_abort_one(struct j1939_priv *priv, struct sk_buff *skb,
                                   bool reverse, bool transmitter)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct j1939_session *session;
        u8 abort = skb->data[1];

        session = j1939_session_get_by_addr(priv, &skcb->addr, reverse,
                                            transmitter);
        if (!session)
                return;

        if (j1939_xtp_rx_cmd_bad_pgn(session, skb))
                goto abort_put;

        netdev_info(priv->ndev, "%s: 0x%p: 0x%05x: (%u) %s\n", __func__,
                    session, j1939_xtp_ctl_to_pgn(skb->data), abort,
                    j1939_xtp_abort_to_str(abort));

        j1939_session_timers_cancel(session);
        session->err = j1939_xtp_abort_to_errno(priv, abort);
        if (session->sk)
                j1939_sk_send_loop_abort(session->sk, session->err);
        else
                j1939_sk_errqueue(session, J1939_ERRQUEUE_RX_ABORT);
        j1939_session_deactivate_activate_next(session);

abort_put:
        j1939_session_put(session);
}

/* abort packets may come in 2 directions */
static void
j1939_xtp_rx_abort(struct j1939_priv *priv, struct sk_buff *skb,
                   bool transmitter)
{
        j1939_xtp_rx_abort_one(priv, skb, false, transmitter);
        j1939_xtp_rx_abort_one(priv, skb, true, transmitter);
}

static void
j1939_xtp_rx_eoma_one(struct j1939_session *session, struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        const u8 *dat;
        int len;

        if (j1939_xtp_rx_cmd_bad_pgn(session, skb))
                return;

        dat = skb->data;

        if (skcb->addr.type == J1939_ETP)
                len = j1939_etp_ctl_to_size(dat);
        else
                len = j1939_tp_ctl_to_size(dat);

        if (session->total_message_size != len) {
                netdev_warn_once(session->priv->ndev,
                                 "%s: 0x%p: Incorrect size. Expected: %i; got: %i.\n",
                                 __func__, session, session->total_message_size,
                                 len);
        }

        netdev_dbg(session->priv->ndev, "%s: 0x%p\n", __func__, session);

        session->pkt.tx_acked = session->pkt.total;
        j1939_session_timers_cancel(session);
        /* transmitted without problems */
        j1939_session_completed(session);
}

static void
j1939_xtp_rx_eoma(struct j1939_priv *priv, struct sk_buff *skb,
                  bool transmitter)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct j1939_session *session;

        session = j1939_session_get_by_addr(priv, &skcb->addr, true,
                                            transmitter);
        if (!session)
                return;

        j1939_xtp_rx_eoma_one(session, skb);
        j1939_session_put(session);
}

static void
j1939_xtp_rx_cts_one(struct j1939_session *session, struct sk_buff *skb)
{
        enum j1939_xtp_abort err = J1939_XTP_ABORT_FAULT;
        unsigned int pkt;
        const u8 *dat;

        dat = skb->data;

        if (j1939_xtp_rx_cmd_bad_pgn(session, skb))
                return;

        netdev_dbg(session->priv->ndev, "%s: 0x%p\n", __func__, session);

        if (session->last_cmd == dat[0]) {
                err = J1939_XTP_ABORT_DUP_SEQ;
                goto out_session_cancel;
        }

        if (session->skcb.addr.type == J1939_ETP)
                pkt = j1939_etp_ctl_to_packet(dat);
        else
                pkt = dat[2];

        if (!pkt)
                goto out_session_cancel;
        else if (dat[1] > session->pkt.block /* 0xff for etp */)
                goto out_session_cancel;

        /* set packet counters only when not CTS(0) */
        session->pkt.tx_acked = pkt - 1;
        j1939_session_skb_drop_old(session);
        session->pkt.last = session->pkt.tx_acked + dat[1];
        if (session->pkt.last > session->pkt.total)
                /* safety measure */
                session->pkt.last = session->pkt.total;
        /* TODO: do not set tx here, do it in txtimer */
        session->pkt.tx = session->pkt.tx_acked;

        session->last_cmd = dat[0];
        if (dat[1]) {
                j1939_tp_set_rxtimeout(session, 1250);
                if (session->transmission) {
                        if (session->pkt.tx_acked)
                                j1939_sk_errqueue(session,
                                                  J1939_ERRQUEUE_TX_SCHED);
                        j1939_session_txtimer_cancel(session);
                        j1939_tp_schedule_txtimer(session, 0);
                }
        } else {
                /* CTS(0) */
                j1939_tp_set_rxtimeout(session, 550);
        }
        return;

 out_session_cancel:
        j1939_session_timers_cancel(session);
        j1939_session_cancel(session, err);
}

static void
j1939_xtp_rx_cts(struct j1939_priv *priv, struct sk_buff *skb, bool transmitter)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct j1939_session *session;

        session = j1939_session_get_by_addr(priv, &skcb->addr, true,
                                            transmitter);
        if (!session)
                return;
        j1939_xtp_rx_cts_one(session, skb);
        j1939_session_put(session);
}

static struct j1939_session *j1939_session_new(struct j1939_priv *priv,
                                               struct sk_buff *skb, size_t size)
{
        struct j1939_session *session;
        struct j1939_sk_buff_cb *skcb;

        session = kzalloc(sizeof(*session), gfp_any());
        if (!session)
                return NULL;

        INIT_LIST_HEAD(&session->active_session_list_entry);
        INIT_LIST_HEAD(&session->sk_session_queue_entry);
        kref_init(&session->kref);

        j1939_priv_get(priv);
        session->priv = priv;
        session->total_message_size = size;
        session->state = J1939_SESSION_NEW;

        skb_queue_head_init(&session->skb_queue);
        skb_queue_tail(&session->skb_queue, skb);

        skcb = j1939_skb_to_cb(skb);
        memcpy(&session->skcb, skcb, sizeof(session->skcb));

        hrtimer_init(&session->txtimer, CLOCK_MONOTONIC,
                     HRTIMER_MODE_REL_SOFT);
        session->txtimer.function = j1939_tp_txtimer;
        hrtimer_init(&session->rxtimer, CLOCK_MONOTONIC,
                     HRTIMER_MODE_REL_SOFT);
        session->rxtimer.function = j1939_tp_rxtimer;

        netdev_dbg(priv->ndev, "%s: 0x%p: sa: %02x, da: %02x\n",
                   __func__, session, skcb->addr.sa, skcb->addr.da);

        return session;
}

static struct
j1939_session *j1939_session_fresh_new(struct j1939_priv *priv,
                                       int size,
                                       const struct j1939_sk_buff_cb *rel_skcb)
{
        struct sk_buff *skb;
        struct j1939_sk_buff_cb *skcb;
        struct j1939_session *session;

        skb = alloc_skb(size + sizeof(struct can_skb_priv), GFP_ATOMIC);
        if (unlikely(!skb))
                return NULL;

        skb->dev = priv->ndev;
        can_skb_reserve(skb);
        can_skb_prv(skb)->ifindex = priv->ndev->ifindex;
        can_skb_prv(skb)->skbcnt = 0;
        skcb = j1939_skb_to_cb(skb);
        memcpy(skcb, rel_skcb, sizeof(*skcb));

        session = j1939_session_new(priv, skb, size);
        if (!session) {
                kfree_skb(skb);
                return NULL;
        }

        /* alloc data area */
        skb_put(skb, size);
        /* skb is recounted in j1939_session_new() */
        return session;
}

int j1939_session_activate(struct j1939_session *session)
{
        struct j1939_priv *priv = session->priv;
        struct j1939_session *active = NULL;
        int ret = 0;

        j1939_session_list_lock(priv);
        if (session->skcb.addr.type != J1939_SIMPLE)
                active = j1939_session_get_by_addr_locked(priv,
                                                          &priv->active_session_list,
                                                          &session->skcb.addr, false,
                                                          session->transmission);
        if (active) {
                j1939_session_put(active);
                ret = -EAGAIN;
        } else {
                WARN_ON_ONCE(session->state != J1939_SESSION_NEW);
                list_add_tail(&session->active_session_list_entry,
                              &priv->active_session_list);
                j1939_session_get(session);
                session->state = J1939_SESSION_ACTIVE;

                netdev_dbg(session->priv->ndev, "%s: 0x%p\n",
                           __func__, session);
        }
        j1939_session_list_unlock(priv);

        return ret;
}

static struct
j1939_session *j1939_xtp_rx_rts_session_new(struct j1939_priv *priv,
                                            struct sk_buff *skb)
{
        enum j1939_xtp_abort abort = J1939_XTP_NO_ABORT;
        struct j1939_sk_buff_cb skcb = *j1939_skb_to_cb(skb);
        struct j1939_session *session;
        const u8 *dat;
        int len, ret;
        pgn_t pgn;

        netdev_dbg(priv->ndev, "%s\n", __func__);

        dat = skb->data;
        pgn = j1939_xtp_ctl_to_pgn(dat);
        skcb.addr.pgn = pgn;

        if (!j1939_sk_recv_match(priv, &skcb))
                return NULL;

        if (skcb.addr.type == J1939_ETP) {
                len = j1939_etp_ctl_to_size(dat);
                if (len > J1939_MAX_ETP_PACKET_SIZE)
                        abort = J1939_XTP_ABORT_FAULT;
                else if (len > priv->tp_max_packet_size)
                        abort = J1939_XTP_ABORT_RESOURCE;
                else if (len <= J1939_MAX_TP_PACKET_SIZE)
                        abort = J1939_XTP_ABORT_FAULT;
        } else {
                len = j1939_tp_ctl_to_size(dat);
                if (len > J1939_MAX_TP_PACKET_SIZE)
                        abort = J1939_XTP_ABORT_FAULT;
                else if (len > priv->tp_max_packet_size)
                        abort = J1939_XTP_ABORT_RESOURCE;
                else if (len < J1939_MIN_TP_PACKET_SIZE)
                        abort = J1939_XTP_ABORT_FAULT;
        }

        if (abort != J1939_XTP_NO_ABORT) {
                j1939_xtp_tx_abort(priv, &skcb, true, abort, pgn);
                return NULL;
        }

        session = j1939_session_fresh_new(priv, len, &skcb);
        if (!session) {
                j1939_xtp_tx_abort(priv, &skcb, true,
                                   J1939_XTP_ABORT_RESOURCE, pgn);
                return NULL;
        }

        /* initialize the control buffer: plain copy */
        session->pkt.total = (len + 6) / 7;
        session->pkt.block = 0xff;
        if (skcb.addr.type != J1939_ETP) {
                if (dat[3] != session->pkt.total)
                        netdev_alert(priv->ndev, "%s: 0x%p: strange total, %u != %u\n",
                                     __func__, session, session->pkt.total,
                                     dat[3]);
                session->pkt.total = dat[3];
                session->pkt.block = min(dat[3], dat[4]);
        }

        session->pkt.rx = 0;
        session->pkt.tx = 0;

        session->tskey = priv->rx_tskey++;
        j1939_sk_errqueue(session, J1939_ERRQUEUE_RX_RTS);

        ret = j1939_session_activate(session);
        if (ret) {
                /* Entering this scope indicates an issue with the J1939 bus.
                 * Possible scenarios include:
                 * - A time lapse occurred, and a new session was initiated
                 *   due to another packet being sent correctly. This could
                 *   have been caused by too long interrupt, debugger, or being
                 *   out-scheduled by another task.
                 * - The bus is receiving numerous erroneous packets, either
                 *   from a malfunctioning device or during a test scenario.
                 */
                netdev_alert(priv->ndev, "%s: 0x%p: concurrent session with same addr (%02x %02x) is already active.\n",
                             __func__, session, skcb.addr.sa, skcb.addr.da);
                j1939_session_put(session);
                return NULL;
        }

        return session;
}

static int j1939_xtp_rx_rts_session_active(struct j1939_session *session,
                                           struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct j1939_priv *priv = session->priv;

        if (!session->transmission) {
                if (j1939_xtp_rx_cmd_bad_pgn(session, skb))
                        return -EBUSY;

                /* RTS on active session */
                j1939_session_timers_cancel(session);
                j1939_session_cancel(session, J1939_XTP_ABORT_BUSY);
        }

        if (session->last_cmd != 0) {
                /* we received a second rts on the same connection */
                netdev_alert(priv->ndev, "%s: 0x%p: connection exists (%02x %02x). last cmd: %x\n",
                             __func__, session, skcb->addr.sa, skcb->addr.da,
                             session->last_cmd);

                j1939_session_timers_cancel(session);
                j1939_session_cancel(session, J1939_XTP_ABORT_BUSY);
                if (session->transmission)
                        j1939_session_deactivate_activate_next(session);

                return -EBUSY;
        }

        if (session->skcb.addr.sa != skcb->addr.sa ||
            session->skcb.addr.da != skcb->addr.da)
                netdev_warn(priv->ndev, "%s: 0x%p: session->skcb.addr.sa=0x%02x skcb->addr.sa=0x%02x session->skcb.addr.da=0x%02x skcb->addr.da=0x%02x\n",
                            __func__, session,
                            session->skcb.addr.sa, skcb->addr.sa,
                            session->skcb.addr.da, skcb->addr.da);
        /* make sure 'sa' & 'da' are correct !
         * They may be 'not filled in yet' for sending
         * skb's, since they did not pass the Address Claim ever.
         */
        session->skcb.addr.sa = skcb->addr.sa;
        session->skcb.addr.da = skcb->addr.da;

        netdev_dbg(session->priv->ndev, "%s: 0x%p\n", __func__, session);

        return 0;
}

static void j1939_xtp_rx_rts(struct j1939_priv *priv, struct sk_buff *skb,
                             bool transmitter)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct j1939_session *session;
        u8 cmd = skb->data[0];

        session = j1939_session_get_by_addr(priv, &skcb->addr, false,
                                            transmitter);

        if (!session) {
                if (transmitter) {
                        /* If we're the transmitter and this function is called,
                         * we received our own RTS. A session has already been
                         * created.
                         *
                         * For some reasons however it might have been destroyed
                         * already. So don't create a new one here (using
                         * "j1939_xtp_rx_rts_session_new()") as this will be a
                         * receiver session.
                         *
                         * The reasons the session is already destroyed might
                         * be:
                         * - user space closed socket was and the session was
                         *   aborted
                         * - session was aborted due to external abort message
                         */
                        return;
                }
                session = j1939_xtp_rx_rts_session_new(priv, skb);
                if (!session) {
                        if (cmd == J1939_TP_CMD_BAM && j1939_sk_recv_match(priv, skcb))
                                netdev_info(priv->ndev, "%s: failed to create TP BAM session\n",
                                            __func__);
                        return;
                }
        } else {
                if (j1939_xtp_rx_rts_session_active(session, skb)) {
                        j1939_session_put(session);
                        return;
                }
        }
        session->last_cmd = cmd;

        if (cmd == J1939_TP_CMD_BAM) {
                if (!session->transmission)
                        j1939_tp_set_rxtimeout(session, 750);
        } else {
                if (!session->transmission) {
                        j1939_session_txtimer_cancel(session);
                        j1939_tp_schedule_txtimer(session, 0);
                }
                j1939_tp_set_rxtimeout(session, 1250);
        }

        j1939_session_put(session);
}

static void j1939_xtp_rx_dpo_one(struct j1939_session *session,
                                 struct sk_buff *skb)
{
        const u8 *dat = skb->data;

        if (j1939_xtp_rx_cmd_bad_pgn(session, skb))
                return;

        netdev_dbg(session->priv->ndev, "%s: 0x%p\n", __func__, session);

        /* transmitted without problems */
        session->pkt.dpo = j1939_etp_ctl_to_packet(skb->data);
        session->last_cmd = dat[0];
        j1939_tp_set_rxtimeout(session, 750);

        if (!session->transmission)
                j1939_sk_errqueue(session, J1939_ERRQUEUE_RX_DPO);
}

static void j1939_xtp_rx_dpo(struct j1939_priv *priv, struct sk_buff *skb,
                             bool transmitter)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct j1939_session *session;

        session = j1939_session_get_by_addr(priv, &skcb->addr, false,
                                            transmitter);
        if (!session) {
                netdev_info(priv->ndev,
                            "%s: no connection found\n", __func__);
                return;
        }

        j1939_xtp_rx_dpo_one(session, skb);
        j1939_session_put(session);
}

static void j1939_xtp_rx_dat_one(struct j1939_session *session,
                                 struct sk_buff *skb)
{
        enum j1939_xtp_abort abort = J1939_XTP_ABORT_FAULT;
        struct j1939_priv *priv = session->priv;
        struct j1939_sk_buff_cb *skcb, *se_skcb;
        struct sk_buff *se_skb = NULL;
        const u8 *dat;
        u8 *tpdat;
        int offset;
        int nbytes;
        bool final = false;
        bool remain = false;
        bool do_cts_eoma = false;
        int packet;

        skcb = j1939_skb_to_cb(skb);
        dat = skb->data;
        if (skb->len != 8) {
                /* makes no sense */
                abort = J1939_XTP_ABORT_UNEXPECTED_DATA;
                goto out_session_cancel;
        }

        switch (session->last_cmd) {
        case 0xff:
                break;
        case J1939_ETP_CMD_DPO:
                if (skcb->addr.type == J1939_ETP)
                        break;
                fallthrough;
        case J1939_TP_CMD_BAM:
                fallthrough;
        case J1939_TP_CMD_CTS:
                if (skcb->addr.type != J1939_ETP)
                        break;
                fallthrough;
        default:
                netdev_info(priv->ndev, "%s: 0x%p: last %02x\n", __func__,
                            session, session->last_cmd);
                goto out_session_cancel;
        }

        packet = (dat[0] - 1 + session->pkt.dpo);
        if (packet > session->pkt.total ||
            (session->pkt.rx + 1) > session->pkt.total) {
                netdev_info(priv->ndev, "%s: 0x%p: should have been completed\n",
                            __func__, session);
                goto out_session_cancel;
        }

        se_skb = j1939_session_skb_get_by_offset(session, packet * 7);
        if (!se_skb) {
                netdev_warn(priv->ndev, "%s: 0x%p: no skb found\n", __func__,
                            session);
                goto out_session_cancel;
        }

        se_skcb = j1939_skb_to_cb(se_skb);
        offset = packet * 7 - se_skcb->offset;
        nbytes = se_skb->len - offset;
        if (nbytes > 7)
                nbytes = 7;
        if (nbytes <= 0 || (nbytes + 1) > skb->len) {
                netdev_info(priv->ndev, "%s: 0x%p: nbytes %i, len %i\n",
                            __func__, session, nbytes, skb->len);
                goto out_session_cancel;
        }

        tpdat = se_skb->data;
        if (!session->transmission) {
                memcpy(&tpdat[offset], &dat[1], nbytes);
        } else {
                int err;

                err = memcmp(&tpdat[offset], &dat[1], nbytes);
                if (err)
                        netdev_err_once(priv->ndev,
                                        "%s: 0x%p: Data of RX-looped back packet (%*ph) doesn't match TX data (%*ph)!\n",
                                        __func__, session,
                                        nbytes, &dat[1],
                                        nbytes, &tpdat[offset]);
        }

        if (packet == session->pkt.rx)
                session->pkt.rx++;

        if (se_skcb->addr.type != J1939_ETP &&
            j1939_cb_is_broadcast(&session->skcb)) {
                if (session->pkt.rx >= session->pkt.total)
                        final = true;
                else
                        remain = true;
        } else {
                /* never final, an EOMA must follow */
                if (session->pkt.rx >= session->pkt.last)
                        do_cts_eoma = true;
        }

        if (final) {
                j1939_session_timers_cancel(session);
                j1939_session_completed(session);
        } else if (remain) {
                if (!session->transmission)
                        j1939_tp_set_rxtimeout(session, 750);
        } else if (do_cts_eoma) {
                j1939_tp_set_rxtimeout(session, 1250);
                if (!session->transmission)
                        j1939_tp_schedule_txtimer(session, 0);
        } else {
                j1939_tp_set_rxtimeout(session, 750);
        }
        session->last_cmd = 0xff;
        consume_skb(se_skb);
        j1939_session_put(session);

        return;

 out_session_cancel:
        kfree_skb(se_skb);
        j1939_session_timers_cancel(session);
        j1939_session_cancel(session, abort);
        j1939_session_put(session);
}

static void j1939_xtp_rx_dat(struct j1939_priv *priv, struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb;
        struct j1939_session *session;

        skcb = j1939_skb_to_cb(skb);

        if (j1939_tp_im_transmitter(skcb)) {
                session = j1939_session_get_by_addr(priv, &skcb->addr, false,
                                                    true);
                if (!session)
                        netdev_info(priv->ndev, "%s: no tx connection found\n",
                                    __func__);
                else
                        j1939_xtp_rx_dat_one(session, skb);
        }

        if (j1939_tp_im_receiver(skcb)) {
                session = j1939_session_get_by_addr(priv, &skcb->addr, false,
                                                    false);
                if (!session)
                        netdev_info(priv->ndev, "%s: no rx connection found\n",
                                    __func__);
                else
                        j1939_xtp_rx_dat_one(session, skb);
        }

        if (j1939_cb_is_broadcast(skcb)) {
                session = j1939_session_get_by_addr(priv, &skcb->addr, false,
                                                    false);
                if (session)
                        j1939_xtp_rx_dat_one(session, skb);
        }
}

/* j1939 main intf */
struct j1939_session *j1939_tp_send(struct j1939_priv *priv,
                                    struct sk_buff *skb, size_t size)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        struct j1939_session *session;
        int ret;

        if (skcb->addr.pgn == J1939_TP_PGN_DAT ||
            skcb->addr.pgn == J1939_TP_PGN_CTL ||
            skcb->addr.pgn == J1939_ETP_PGN_DAT ||
            skcb->addr.pgn == J1939_ETP_PGN_CTL)
                /* avoid conflict */
                return ERR_PTR(-EDOM);

        if (size > priv->tp_max_packet_size)
                return ERR_PTR(-EMSGSIZE);

        if (size <= 8)
                skcb->addr.type = J1939_SIMPLE;
        else if (size > J1939_MAX_TP_PACKET_SIZE)
                skcb->addr.type = J1939_ETP;
        else
                skcb->addr.type = J1939_TP;

        if (skcb->addr.type == J1939_ETP &&
            j1939_cb_is_broadcast(skcb))
                return ERR_PTR(-EDESTADDRREQ);

        /* fill in addresses from names */
        ret = j1939_ac_fixup(priv, skb);
        if (unlikely(ret))
                return ERR_PTR(ret);

        /* fix DST flags, it may be used there soon */
        if (j1939_address_is_unicast(skcb->addr.da) &&
            priv->ents[skcb->addr.da].nusers)
                skcb->flags |= J1939_ECU_LOCAL_DST;

        /* src is always local, I'm sending ... */
        skcb->flags |= J1939_ECU_LOCAL_SRC;

        /* prepare new session */
        session = j1939_session_new(priv, skb, size);
        if (!session)
                return ERR_PTR(-ENOMEM);

        /* skb is recounted in j1939_session_new() */
        sock_hold(skb->sk);
        session->sk = skb->sk;
        session->transmission = true;
        session->pkt.total = (size + 6) / 7;
        session->pkt.block = skcb->addr.type == J1939_ETP ? 255 :
                min(j1939_tp_block ?: 255, session->pkt.total);

        if (j1939_cb_is_broadcast(&session->skcb))
                /* set the end-packet for broadcast */
                session->pkt.last = session->pkt.total;

        skcb->tskey = atomic_inc_return(&session->sk->sk_tskey) - 1;
        session->tskey = skcb->tskey;

        return session;
}

static void j1939_tp_cmd_recv(struct j1939_priv *priv, struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);
        int extd = J1939_TP;
        u8 cmd = skb->data[0];

        switch (cmd) {
        case J1939_ETP_CMD_RTS:
                extd = J1939_ETP;
                fallthrough;
        case J1939_TP_CMD_BAM:
                if (cmd == J1939_TP_CMD_BAM && !j1939_cb_is_broadcast(skcb)) {
                        netdev_err_once(priv->ndev, "%s: BAM to unicast (%02x), ignoring!\n",
                                        __func__, skcb->addr.sa);
                        return;
                }
                fallthrough;
        case J1939_TP_CMD_RTS:
                if (skcb->addr.type != extd)
                        return;

                if (cmd == J1939_TP_CMD_RTS && j1939_cb_is_broadcast(skcb)) {
                        netdev_alert(priv->ndev, "%s: rts without destination (%02x)\n",
                                     __func__, skcb->addr.sa);
                        return;
                }

                if (j1939_tp_im_transmitter(skcb))
                        j1939_xtp_rx_rts(priv, skb, true);

                if (j1939_tp_im_receiver(skcb) || j1939_cb_is_broadcast(skcb))
                        j1939_xtp_rx_rts(priv, skb, false);

                break;

        case J1939_ETP_CMD_CTS:
                extd = J1939_ETP;
                fallthrough;
        case J1939_TP_CMD_CTS:
                if (skcb->addr.type != extd)
                        return;

                if (j1939_tp_im_transmitter(skcb))
                        j1939_xtp_rx_cts(priv, skb, false);

                if (j1939_tp_im_receiver(skcb))
                        j1939_xtp_rx_cts(priv, skb, true);

                break;

        case J1939_ETP_CMD_DPO:
                if (skcb->addr.type != J1939_ETP)
                        return;

                if (j1939_tp_im_transmitter(skcb))
                        j1939_xtp_rx_dpo(priv, skb, true);

                if (j1939_tp_im_receiver(skcb))
                        j1939_xtp_rx_dpo(priv, skb, false);

                break;

        case J1939_ETP_CMD_EOMA:
                extd = J1939_ETP;
                fallthrough;
        case J1939_TP_CMD_EOMA:
                if (skcb->addr.type != extd)
                        return;

                if (j1939_tp_im_transmitter(skcb))
                        j1939_xtp_rx_eoma(priv, skb, false);

                if (j1939_tp_im_receiver(skcb))
                        j1939_xtp_rx_eoma(priv, skb, true);

                break;

        case J1939_ETP_CMD_ABORT: /* && J1939_TP_CMD_ABORT */
                if (j1939_cb_is_broadcast(skcb)) {
                        netdev_err_once(priv->ndev, "%s: abort to broadcast (%02x), ignoring!\n",
                                        __func__, skcb->addr.sa);
                        return;
                }

                if (j1939_tp_im_transmitter(skcb))
                        j1939_xtp_rx_abort(priv, skb, true);

                if (j1939_tp_im_receiver(skcb))
                        j1939_xtp_rx_abort(priv, skb, false);

                break;
        default:
                return;
        }
}

int j1939_tp_recv(struct j1939_priv *priv, struct sk_buff *skb)
{
        struct j1939_sk_buff_cb *skcb = j1939_skb_to_cb(skb);

        if (!j1939_tp_im_involved_anydir(skcb) && !j1939_cb_is_broadcast(skcb))
                return 0;

        switch (skcb->addr.pgn) {
        case J1939_ETP_PGN_DAT:
                skcb->addr.type = J1939_ETP;
                fallthrough;
        case J1939_TP_PGN_DAT:
                j1939_xtp_rx_dat(priv, skb);
                break;

        case J1939_ETP_PGN_CTL:
                skcb->addr.type = J1939_ETP;
                fallthrough;
        case J1939_TP_PGN_CTL:
                if (skb->len < 8)
                        return 0; /* Don't care. Nothing to extract here */

                j1939_tp_cmd_recv(priv, skb);
                break;
        default:
                return 0; /* no problem */
        }
        return 1; /* "I processed the message" */
}

void j1939_simple_recv(struct j1939_priv *priv, struct sk_buff *skb)
{
        struct j1939_session *session;

        if (!skb->sk)
                return;

        if (skb->sk->sk_family != AF_CAN ||
            skb->sk->sk_protocol != CAN_J1939)
                return;

        j1939_session_list_lock(priv);
        session = j1939_session_get_simple(priv, skb);
        j1939_session_list_unlock(priv);
        if (!session) {
                netdev_warn(priv->ndev,
                            "%s: Received already invalidated message\n",
                            __func__);
                return;
        }

        j1939_session_timers_cancel(session);
        j1939_session_deactivate(session);
        j1939_session_put(session);
}

int j1939_cancel_active_session(struct j1939_priv *priv, struct sock *sk)
{
        struct j1939_session *session, *saved;

        netdev_dbg(priv->ndev, "%s, sk: %p\n", __func__, sk);
        j1939_session_list_lock(priv);
        list_for_each_entry_safe(session, saved,
                                 &priv->active_session_list,
                                 active_session_list_entry) {
                if (!sk || sk == session->sk) {
                        if (hrtimer_try_to_cancel(&session->txtimer) == 1)
                                j1939_session_put(session);
                        if (hrtimer_try_to_cancel(&session->rxtimer) == 1)
                                j1939_session_put(session);

                        session->err = ESHUTDOWN;
                        j1939_session_deactivate_locked(session);
                }
        }
        j1939_session_list_unlock(priv);
        return NOTIFY_DONE;
}

void j1939_tp_init(struct j1939_priv *priv)
{
        spin_lock_init(&priv->active_session_list_lock);
        INIT_LIST_HEAD(&priv->active_session_list);
        priv->tp_max_packet_size = J1939_MAX_ETP_PACKET_SIZE;
}


















    2 











































































































































































































































   20 



 1230 














































































  466 






















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_NETLINK_H
#define __LINUX_NETLINK_H


#include <linux/capability.h>
#include <linux/skbuff.h>
#include <linux/export.h>
#include <net/scm.h>
#include <uapi/linux/netlink.h>

struct net;

void do_trace_netlink_extack(const char *msg);

static inline struct nlmsghdr *nlmsg_hdr(const struct sk_buff *skb)
{
        return (struct nlmsghdr *)skb->data;
}

enum netlink_skb_flags {
        NETLINK_SKB_DST                = 0x8,        /* Dst set in sendto or sendmsg */
};

struct netlink_skb_parms {
        struct scm_creds        creds;                /* Skb credentials        */
        __u32                        portid;
        __u32                        dst_group;
        __u32                        flags;
        struct sock                *sk;
        bool                        nsid_is_set;
        int                        nsid;
};

#define NETLINK_CB(skb)                (*(struct netlink_skb_parms*)&((skb)->cb))
#define NETLINK_CREDS(skb)        (&NETLINK_CB((skb)).creds)


void netlink_table_grab(void);
void netlink_table_ungrab(void);

#define NL_CFG_F_NONROOT_RECV        (1 << 0)
#define NL_CFG_F_NONROOT_SEND        (1 << 1)

/* optional Netlink kernel configuration parameters */
struct netlink_kernel_cfg {
        unsigned int        groups;
        unsigned int        flags;
        void                (*input)(struct sk_buff *skb);
        int                (*bind)(struct net *net, int group);
        void                (*unbind)(struct net *net, int group);
        void            (*release) (struct sock *sk, unsigned long *groups);
};

struct sock *__netlink_kernel_create(struct net *net, int unit,
                                            struct module *module,
                                            struct netlink_kernel_cfg *cfg);
static inline struct sock *
netlink_kernel_create(struct net *net, int unit, struct netlink_kernel_cfg *cfg)
{
        return __netlink_kernel_create(net, unit, THIS_MODULE, cfg);
}

/* this can be increased when necessary - don't expose to userland */
#define NETLINK_MAX_COOKIE_LEN        20
#define NETLINK_MAX_FMTMSG_LEN        80

/**
 * struct netlink_ext_ack - netlink extended ACK report struct
 * @_msg: message string to report - don't access directly, use
 *        %NL_SET_ERR_MSG
 * @bad_attr: attribute with error
 * @policy: policy for a bad attribute
 * @miss_type: attribute type which was missing
 * @miss_nest: nest missing an attribute (%NULL if missing top level attr)
 * @cookie: cookie data to return to userspace (for success)
 * @cookie_len: actual cookie data length
 * @_msg_buf: output buffer for formatted message strings - don't access
 *        directly, use %NL_SET_ERR_MSG_FMT
 */
struct netlink_ext_ack {
        const char *_msg;
        const struct nlattr *bad_attr;
        const struct nla_policy *policy;
        const struct nlattr *miss_nest;
        u16 miss_type;
        u8 cookie[NETLINK_MAX_COOKIE_LEN];
        u8 cookie_len;
        char _msg_buf[NETLINK_MAX_FMTMSG_LEN];
};

/* Always use this macro, this allows later putting the
 * message into a separate section or such for things
 * like translation or listing all possible messages.
 * If string formatting is needed use NL_SET_ERR_MSG_FMT.
 */
#define NL_SET_ERR_MSG(extack, msg) do {                \
        static const char __msg[] = msg;                \
        struct netlink_ext_ack *__extack = (extack);        \
                                                        \
        do_trace_netlink_extack(__msg);                        \
                                                        \
        if (__extack)                                        \
                __extack->_msg = __msg;                        \
} while (0)

/* We splice fmt with %s at each end even in the snprintf so that both calls
 * can use the same string constant, avoiding its duplication in .ro
 */
#define NL_SET_ERR_MSG_FMT(extack, fmt, args...) do {                               \
        struct netlink_ext_ack *__extack = (extack);                               \
                                                                               \
        if (!__extack)                                                               \
                break;                                                               \
        if (snprintf(__extack->_msg_buf, NETLINK_MAX_FMTMSG_LEN,               \
                     "%s" fmt "%s", "", ##args, "") >=                               \
            NETLINK_MAX_FMTMSG_LEN)                                               \
                net_warn_ratelimited("%s" fmt "%s", "truncated extack: ",      \
                                     ##args, "\n");                               \
                                                                               \
        do_trace_netlink_extack(__extack->_msg_buf);                               \
                                                                               \
        __extack->_msg = __extack->_msg_buf;                                       \
} while (0)

#define NL_SET_ERR_MSG_MOD(extack, msg)                        \
        NL_SET_ERR_MSG((extack), KBUILD_MODNAME ": " msg)

#define NL_SET_ERR_MSG_FMT_MOD(extack, fmt, args...)        \
        NL_SET_ERR_MSG_FMT((extack), KBUILD_MODNAME ": " fmt, ##args)

#define NL_SET_ERR_MSG_WEAK(extack, msg) do {                \
        if ((extack) && !(extack)->_msg)                \
                NL_SET_ERR_MSG((extack), msg);                \
} while (0)

#define NL_SET_ERR_MSG_WEAK_MOD(extack, msg) do {        \
        if ((extack) && !(extack)->_msg)                \
                NL_SET_ERR_MSG_MOD((extack), msg);        \
} while (0)

#define NL_SET_BAD_ATTR_POLICY(extack, attr, pol) do {        \
        if ((extack)) {                                        \
                (extack)->bad_attr = (attr);                \
                (extack)->policy = (pol);                \
        }                                                \
} while (0)

#define NL_SET_BAD_ATTR(extack, attr) NL_SET_BAD_ATTR_POLICY(extack, attr, NULL)

#define NL_SET_ERR_MSG_ATTR_POL(extack, attr, pol, msg) do {        \
        static const char __msg[] = msg;                        \
        struct netlink_ext_ack *__extack = (extack);                \
                                                                \
        do_trace_netlink_extack(__msg);                                \
                                                                \
        if (__extack) {                                                \
                __extack->_msg = __msg;                                \
                __extack->bad_attr = (attr);                        \
                __extack->policy = (pol);                        \
        }                                                        \
} while (0)

#define NL_SET_ERR_MSG_ATTR_POL_FMT(extack, attr, pol, fmt, args...) do {        \
        struct netlink_ext_ack *__extack = (extack);                                \
                                                                                \
        if (!__extack)                                                                \
                break;                                                                \
                                                                                \
        if (snprintf(__extack->_msg_buf, NETLINK_MAX_FMTMSG_LEN,                \
                     "%s" fmt "%s", "", ##args, "") >=                                \
            NETLINK_MAX_FMTMSG_LEN)                                                \
                net_warn_ratelimited("%s" fmt "%s", "truncated extack: ",       \
                                     ##args, "\n");                                \
                                                                                \
        do_trace_netlink_extack(__extack->_msg_buf);                                \
                                                                                \
        __extack->_msg = __extack->_msg_buf;                                        \
        __extack->bad_attr = (attr);                                                \
        __extack->policy = (pol);                                                \
} while (0)

#define NL_SET_ERR_MSG_ATTR(extack, attr, msg)                \
        NL_SET_ERR_MSG_ATTR_POL(extack, attr, NULL, msg)

#define NL_SET_ERR_MSG_ATTR_FMT(extack, attr, msg, args...) \
        NL_SET_ERR_MSG_ATTR_POL_FMT(extack, attr, NULL, msg, ##args)

#define NL_SET_ERR_ATTR_MISS(extack, nest, type)  do {        \
        struct netlink_ext_ack *__extack = (extack);        \
                                                        \
        if (__extack) {                                        \
                __extack->miss_nest = (nest);                \
                __extack->miss_type = (type);                \
        }                                                \
} while (0)

#define NL_REQ_ATTR_CHECK(extack, nest, tb, type) ({                \
        struct nlattr **__tb = (tb);                                \
        u32 __attr = (type);                                        \
        int __retval;                                                \
                                                                \
        __retval = !__tb[__attr];                                \
        if (__retval)                                                \
                NL_SET_ERR_ATTR_MISS((extack), (nest), __attr);        \
        __retval;                                                \
})

static inline void nl_set_extack_cookie_u64(struct netlink_ext_ack *extack,
                                            u64 cookie)
{
        if (!extack)
                return;
        memcpy(extack->cookie, &cookie, sizeof(cookie));
        extack->cookie_len = sizeof(cookie);
}

void netlink_kernel_release(struct sock *sk);
int __netlink_change_ngroups(struct sock *sk, unsigned int groups);
int netlink_change_ngroups(struct sock *sk, unsigned int groups);
void __netlink_clear_multicast_users(struct sock *sk, unsigned int group);
void netlink_ack(struct sk_buff *in_skb, struct nlmsghdr *nlh, int err,
                 const struct netlink_ext_ack *extack);
int netlink_has_listeners(struct sock *sk, unsigned int group);
bool netlink_strict_get_check(struct sk_buff *skb);

int netlink_unicast(struct sock *ssk, struct sk_buff *skb, __u32 portid, int nonblock);
int netlink_broadcast(struct sock *ssk, struct sk_buff *skb, __u32 portid,
                      __u32 group, gfp_t allocation);

typedef int (*netlink_filter_fn)(struct sock *dsk, struct sk_buff *skb, void *data);

int netlink_broadcast_filtered(struct sock *ssk, struct sk_buff *skb,
                               __u32 portid, __u32 group, gfp_t allocation,
                               netlink_filter_fn filter,
                               void *filter_data);
int netlink_set_err(struct sock *ssk, __u32 portid, __u32 group, int code);
int netlink_register_notifier(struct notifier_block *nb);
int netlink_unregister_notifier(struct notifier_block *nb);

/* finegrained unicast helpers: */
struct sock *netlink_getsockbyfilp(struct file *filp);
int netlink_attachskb(struct sock *sk, struct sk_buff *skb,
                      long *timeo, struct sock *ssk);
void netlink_detachskb(struct sock *sk, struct sk_buff *skb);
int netlink_sendskb(struct sock *sk, struct sk_buff *skb);

static inline struct sk_buff *
netlink_skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
{
        struct sk_buff *nskb;

        nskb = skb_clone(skb, gfp_mask);
        if (!nskb)
                return NULL;

        /* This is a large skb, set destructor callback to release head */
        if (is_vmalloc_addr(skb->head))
                nskb->destructor = skb->destructor;

        return nskb;
}

/*
 *        skb should fit one page. This choice is good for headerless malloc.
 *        But we should limit to 8K so that userspace does not have to
 *        use enormous buffer sizes on recvmsg() calls just to avoid
 *        MSG_TRUNC when PAGE_SIZE is very large.
 */
#if PAGE_SIZE < 8192UL
#define NLMSG_GOODSIZE        SKB_WITH_OVERHEAD(PAGE_SIZE)
#else
#define NLMSG_GOODSIZE        SKB_WITH_OVERHEAD(8192UL)
#endif

#define NLMSG_DEFAULT_SIZE (NLMSG_GOODSIZE - NLMSG_HDRLEN)


struct netlink_callback {
        struct sk_buff                *skb;
        const struct nlmsghdr        *nlh;
        int                        (*dump)(struct sk_buff * skb,
                                        struct netlink_callback *cb);
        int                        (*done)(struct netlink_callback *cb);
        void                        *data;
        /* the module that dump function belong to */
        struct module                *module;
        struct netlink_ext_ack        *extack;
        u16                        family;
        u16                        answer_flags;
        u32                        min_dump_alloc;
        unsigned int                prev_seq, seq;
        int                        flags;
        bool                        strict_check;
        union {
                u8                ctx[48];

                /* args is deprecated. Cast a struct over ctx instead
                 * for proper type safety.
                 */
                long                args[6];
        };
};

#define NL_ASSERT_DUMP_CTX_FITS(type_name)                                \
        BUILD_BUG_ON(sizeof(type_name) >                                \
                     sizeof_field(struct netlink_callback, ctx))

struct netlink_notify {
        struct net *net;
        u32 portid;
        int protocol;
};

struct nlmsghdr *
__nlmsg_put(struct sk_buff *skb, u32 portid, u32 seq, int type, int len, int flags);

struct netlink_dump_control {
        int (*start)(struct netlink_callback *);
        int (*dump)(struct sk_buff *skb, struct netlink_callback *);
        int (*done)(struct netlink_callback *);
        struct netlink_ext_ack *extack;
        void *data;
        struct module *module;
        u32 min_dump_alloc;
        int flags;
};

int __netlink_dump_start(struct sock *ssk, struct sk_buff *skb,
                                const struct nlmsghdr *nlh,
                                struct netlink_dump_control *control);
static inline int netlink_dump_start(struct sock *ssk, struct sk_buff *skb,
                                     const struct nlmsghdr *nlh,
                                     struct netlink_dump_control *control)
{
        if (!control->module)
                control->module = THIS_MODULE;

        return __netlink_dump_start(ssk, skb, nlh, control);
}

struct netlink_tap {
        struct net_device *dev;
        struct module *module;
        struct list_head list;
};

int netlink_add_tap(struct netlink_tap *nt);
int netlink_remove_tap(struct netlink_tap *nt);

bool __netlink_ns_capable(const struct netlink_skb_parms *nsp,
                          struct user_namespace *ns, int cap);
bool netlink_ns_capable(const struct sk_buff *skb,
                        struct user_namespace *ns, int cap);
bool netlink_capable(const struct sk_buff *skb, int cap);
bool netlink_net_capable(const struct sk_buff *skb, int cap);
struct sk_buff *netlink_alloc_large_skb(unsigned int size, int broadcast);

#endif        /* __LINUX_NETLINK_H */


































































































    1 
















    1 











    1 




































































































































































































































    2 

























































    2 






















    2 





























    2 



























































    7 
    5 



    2 




























































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Backlight Lowlevel Control Abstraction
 *
 * Copyright (C) 2003,2004 Hewlett-Packard Company
 *
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/init.h>
#include <linux/device.h>
#include <linux/backlight.h>
#include <linux/notifier.h>
#include <linux/ctype.h>
#include <linux/err.h>
#include <linux/fb.h>
#include <linux/slab.h>

#ifdef CONFIG_PMAC_BACKLIGHT
#include <asm/backlight.h>
#endif

/**
 * DOC: overview
 *
 * The backlight core supports implementing backlight drivers.
 *
 * A backlight driver registers a driver using
 * devm_backlight_device_register(). The properties of the backlight
 * driver such as type and max_brightness must be specified.
 * When the core detect changes in for example brightness or power state
 * the update_status() operation is called. The backlight driver shall
 * implement this operation and use it to adjust backlight.
 *
 * Several sysfs attributes are provided by the backlight core::
 *
 * - brightness         R/W, set the requested brightness level
 * - actual_brightness  RO, the brightness level used by the HW
 * - max_brightness     RO, the maximum  brightness level supported
 *
 * See Documentation/ABI/stable/sysfs-class-backlight for the full list.
 *
 * The backlight can be adjusted using the sysfs interface, and
 * the backlight driver may also support adjusting backlight using
 * a hot-key or some other platform or firmware specific way.
 *
 * The driver must implement the get_brightness() operation if
 * the HW do not support all the levels that can be specified in
 * brightness, thus providing user-space access to the actual level
 * via the actual_brightness attribute.
 *
 * When the backlight changes this is reported to user-space using
 * an uevent connected to the actual_brightness attribute.
 * When brightness is set by platform specific means, for example
 * a hot-key to adjust backlight, the driver must notify the backlight
 * core that brightness has changed using backlight_force_update().
 *
 * The backlight driver core receives notifications from fbdev and
 * if the event is FB_EVENT_BLANK and if the value of blank, from the
 * FBIOBLANK ioctrl, results in a change in the backlight state the
 * update_status() operation is called.
 */

static struct list_head backlight_dev_list;
static struct mutex backlight_dev_list_mutex;
static struct blocking_notifier_head backlight_notifier;

static const char *const backlight_types[] = {
        [BACKLIGHT_RAW] = "raw",
        [BACKLIGHT_PLATFORM] = "platform",
        [BACKLIGHT_FIRMWARE] = "firmware",
};

static const char *const backlight_scale_types[] = {
        [BACKLIGHT_SCALE_UNKNOWN]        = "unknown",
        [BACKLIGHT_SCALE_LINEAR]        = "linear",
        [BACKLIGHT_SCALE_NON_LINEAR]        = "non-linear",
};

#if defined(CONFIG_FB_CORE) || (defined(CONFIG_FB_CORE_MODULE) && \
                                defined(CONFIG_BACKLIGHT_CLASS_DEVICE_MODULE))
/*
 * fb_notifier_callback
 *
 * This callback gets called when something important happens inside a
 * framebuffer driver. The backlight core only cares about FB_BLANK_UNBLANK
 * which is reported to the driver using backlight_update_status()
 * as a state change.
 *
 * There may be several fbdev's connected to the backlight device,
 * in which case they are kept track of. A state change is only reported
 * if there is a change in backlight for the specified fbdev.
 */
static int fb_notifier_callback(struct notifier_block *self,
                                unsigned long event, void *data)
{
        struct backlight_device *bd;
        struct fb_event *evdata = data;
        struct fb_info *info = evdata->info;
        struct backlight_device *fb_bd = fb_bl_device(info);
        int node = info->node;
        int fb_blank = 0;

        /* If we aren't interested in this event, skip it immediately ... */
        if (event != FB_EVENT_BLANK)
                return 0;

        bd = container_of(self, struct backlight_device, fb_notif);
        mutex_lock(&bd->ops_lock);

        if (!bd->ops)
                goto out;
        if (bd->ops->controls_device && !bd->ops->controls_device(bd, info->device))
                goto out;
        if (fb_bd && fb_bd != bd)
                goto out;

        fb_blank = *(int *)evdata->data;
        if (fb_blank == FB_BLANK_UNBLANK && !bd->fb_bl_on[node]) {
                bd->fb_bl_on[node] = true;
                if (!bd->use_count++) {
                        bd->props.state &= ~BL_CORE_FBBLANK;
                        backlight_update_status(bd);
                }
        } else if (fb_blank != FB_BLANK_UNBLANK && bd->fb_bl_on[node]) {
                bd->fb_bl_on[node] = false;
                if (!(--bd->use_count)) {
                        bd->props.state |= BL_CORE_FBBLANK;
                        backlight_update_status(bd);
                }
        }
out:
        mutex_unlock(&bd->ops_lock);
        return 0;
}

static int backlight_register_fb(struct backlight_device *bd)
{
        memset(&bd->fb_notif, 0, sizeof(bd->fb_notif));
        bd->fb_notif.notifier_call = fb_notifier_callback;

        return fb_register_client(&bd->fb_notif);
}

static void backlight_unregister_fb(struct backlight_device *bd)
{
        fb_unregister_client(&bd->fb_notif);
}
#else
static inline int backlight_register_fb(struct backlight_device *bd)
{
        return 0;
}

static inline void backlight_unregister_fb(struct backlight_device *bd)
{
}
#endif /* CONFIG_FB_CORE */

static void backlight_generate_event(struct backlight_device *bd,
                                     enum backlight_update_reason reason)
{
        char *envp[2];

        switch (reason) {
        case BACKLIGHT_UPDATE_SYSFS:
                envp[0] = "SOURCE=sysfs";
                break;
        case BACKLIGHT_UPDATE_HOTKEY:
                envp[0] = "SOURCE=hotkey";
                break;
        default:
                envp[0] = "SOURCE=unknown";
                break;
        }
        envp[1] = NULL;
        kobject_uevent_env(&bd->dev.kobj, KOBJ_CHANGE, envp);
        sysfs_notify(&bd->dev.kobj, NULL, "actual_brightness");
}

static ssize_t bl_power_show(struct device *dev, struct device_attribute *attr,
                char *buf)
{
        struct backlight_device *bd = to_backlight_device(dev);

        return sprintf(buf, "%d\n", bd->props.power);
}

static ssize_t bl_power_store(struct device *dev, struct device_attribute *attr,
                const char *buf, size_t count)
{
        int rc;
        struct backlight_device *bd = to_backlight_device(dev);
        unsigned long power, old_power;

        rc = kstrtoul(buf, 0, &power);
        if (rc)
                return rc;

        rc = -ENXIO;
        mutex_lock(&bd->ops_lock);
        if (bd->ops) {
                pr_debug("set power to %lu\n", power);
                if (bd->props.power != power) {
                        old_power = bd->props.power;
                        bd->props.power = power;
                        rc = backlight_update_status(bd);
                        if (rc)
                                bd->props.power = old_power;
                        else
                                rc = count;
                } else {
                        rc = count;
                }
        }
        mutex_unlock(&bd->ops_lock);

        return rc;
}
static DEVICE_ATTR_RW(bl_power);

static ssize_t brightness_show(struct device *dev,
                struct device_attribute *attr, char *buf)
{
        struct backlight_device *bd = to_backlight_device(dev);

        return sprintf(buf, "%d\n", bd->props.brightness);
}

int backlight_device_set_brightness(struct backlight_device *bd,
                                    unsigned long brightness)
{
        int rc = -ENXIO;

        mutex_lock(&bd->ops_lock);
        if (bd->ops) {
                if (brightness > bd->props.max_brightness)
                        rc = -EINVAL;
                else {
                        pr_debug("set brightness to %lu\n", brightness);
                        bd->props.brightness = brightness;
                        rc = backlight_update_status(bd);
                }
        }
        mutex_unlock(&bd->ops_lock);

        backlight_generate_event(bd, BACKLIGHT_UPDATE_SYSFS);

        return rc;
}
EXPORT_SYMBOL(backlight_device_set_brightness);

static ssize_t brightness_store(struct device *dev,
                struct device_attribute *attr, const char *buf, size_t count)
{
        int rc;
        struct backlight_device *bd = to_backlight_device(dev);
        unsigned long brightness;

        rc = kstrtoul(buf, 0, &brightness);
        if (rc)
                return rc;

        rc = backlight_device_set_brightness(bd, brightness);

        return rc ? rc : count;
}
static DEVICE_ATTR_RW(brightness);

static ssize_t type_show(struct device *dev, struct device_attribute *attr,
                char *buf)
{
        struct backlight_device *bd = to_backlight_device(dev);

        return sprintf(buf, "%s\n", backlight_types[bd->props.type]);
}
static DEVICE_ATTR_RO(type);

static ssize_t max_brightness_show(struct device *dev,
                struct device_attribute *attr, char *buf)
{
        struct backlight_device *bd = to_backlight_device(dev);

        return sprintf(buf, "%d\n", bd->props.max_brightness);
}
static DEVICE_ATTR_RO(max_brightness);

static ssize_t actual_brightness_show(struct device *dev,
                struct device_attribute *attr, char *buf)
{
        int rc = -ENXIO;
        struct backlight_device *bd = to_backlight_device(dev);

        mutex_lock(&bd->ops_lock);
        if (bd->ops && bd->ops->get_brightness) {
                rc = bd->ops->get_brightness(bd);
                if (rc >= 0)
                        rc = sprintf(buf, "%d\n", rc);
        } else {
                rc = sprintf(buf, "%d\n", bd->props.brightness);
        }
        mutex_unlock(&bd->ops_lock);

        return rc;
}
static DEVICE_ATTR_RO(actual_brightness);

static ssize_t scale_show(struct device *dev,
                struct device_attribute *attr, char *buf)
{
        struct backlight_device *bd = to_backlight_device(dev);

        if (WARN_ON(bd->props.scale > BACKLIGHT_SCALE_NON_LINEAR))
                return sprintf(buf, "unknown\n");

        return sprintf(buf, "%s\n", backlight_scale_types[bd->props.scale]);
}
static DEVICE_ATTR_RO(scale);

#ifdef CONFIG_PM_SLEEP
static int backlight_suspend(struct device *dev)
{
        struct backlight_device *bd = to_backlight_device(dev);

        mutex_lock(&bd->ops_lock);
        if (bd->ops && bd->ops->options & BL_CORE_SUSPENDRESUME) {
                bd->props.state |= BL_CORE_SUSPENDED;
                backlight_update_status(bd);
        }
        mutex_unlock(&bd->ops_lock);

        return 0;
}

static int backlight_resume(struct device *dev)
{
        struct backlight_device *bd = to_backlight_device(dev);

        mutex_lock(&bd->ops_lock);
        if (bd->ops && bd->ops->options & BL_CORE_SUSPENDRESUME) {
                bd->props.state &= ~BL_CORE_SUSPENDED;
                backlight_update_status(bd);
        }
        mutex_unlock(&bd->ops_lock);

        return 0;
}
#endif

static SIMPLE_DEV_PM_OPS(backlight_class_dev_pm_ops, backlight_suspend,
                         backlight_resume);

static void bl_device_release(struct device *dev)
{
        struct backlight_device *bd = to_backlight_device(dev);
        kfree(bd);
}

static struct attribute *bl_device_attrs[] = {
        &dev_attr_bl_power.attr,
        &dev_attr_brightness.attr,
        &dev_attr_actual_brightness.attr,
        &dev_attr_max_brightness.attr,
        &dev_attr_scale.attr,
        &dev_attr_type.attr,
        NULL,
};
ATTRIBUTE_GROUPS(bl_device);

static const struct class backlight_class = {
        .name = "backlight",
        .dev_groups = bl_device_groups,
        .pm = &backlight_class_dev_pm_ops,
};

/**
 * backlight_force_update - tell the backlight subsystem that hardware state
 *   has changed
 * @bd: the backlight device to update
 * @reason: reason for update
 *
 * Updates the internal state of the backlight in response to a hardware event,
 * and generates an uevent to notify userspace. A backlight driver shall call
 * backlight_force_update() when the backlight is changed using, for example,
 * a hot-key. The updated brightness is read using get_brightness() and the
 * brightness value is reported using an uevent.
 */
void backlight_force_update(struct backlight_device *bd,
                            enum backlight_update_reason reason)
{
        int brightness;

        mutex_lock(&bd->ops_lock);
        if (bd->ops && bd->ops->get_brightness) {
                brightness = bd->ops->get_brightness(bd);
                if (brightness >= 0)
                        bd->props.brightness = brightness;
                else
                        dev_err(&bd->dev,
                                "Could not update brightness from device: %pe\n",
                                ERR_PTR(brightness));
        }
        mutex_unlock(&bd->ops_lock);
        backlight_generate_event(bd, reason);
}
EXPORT_SYMBOL(backlight_force_update);

/* deprecated - use devm_backlight_device_register() */
struct backlight_device *backlight_device_register(const char *name,
        struct device *parent, void *devdata, const struct backlight_ops *ops,
        const struct backlight_properties *props)
{
        struct backlight_device *new_bd;
        int rc;

        pr_debug("backlight_device_register: name=%s\n", name);

        new_bd = kzalloc(sizeof(struct backlight_device), GFP_KERNEL);
        if (!new_bd)
                return ERR_PTR(-ENOMEM);

        mutex_init(&new_bd->update_lock);
        mutex_init(&new_bd->ops_lock);

        new_bd->dev.class = &backlight_class;
        new_bd->dev.parent = parent;
        new_bd->dev.release = bl_device_release;
        dev_set_name(&new_bd->dev, "%s", name);
        dev_set_drvdata(&new_bd->dev, devdata);

        /* Set default properties */
        if (props) {
                memcpy(&new_bd->props, props,
                       sizeof(struct backlight_properties));
                if (props->type <= 0 || props->type >= BACKLIGHT_TYPE_MAX) {
                        WARN(1, "%s: invalid backlight type", name);
                        new_bd->props.type = BACKLIGHT_RAW;
                }
        } else {
                new_bd->props.type = BACKLIGHT_RAW;
        }

        rc = device_register(&new_bd->dev);
        if (rc) {
                put_device(&new_bd->dev);
                return ERR_PTR(rc);
        }

        rc = backlight_register_fb(new_bd);
        if (rc) {
                device_unregister(&new_bd->dev);
                return ERR_PTR(rc);
        }

        new_bd->ops = ops;

#ifdef CONFIG_PMAC_BACKLIGHT
        mutex_lock(&pmac_backlight_mutex);
        if (!pmac_backlight)
                pmac_backlight = new_bd;
        mutex_unlock(&pmac_backlight_mutex);
#endif

        mutex_lock(&backlight_dev_list_mutex);
        list_add(&new_bd->entry, &backlight_dev_list);
        mutex_unlock(&backlight_dev_list_mutex);

        blocking_notifier_call_chain(&backlight_notifier,
                                     BACKLIGHT_REGISTERED, new_bd);

        return new_bd;
}
EXPORT_SYMBOL(backlight_device_register);

/** backlight_device_get_by_type - find first backlight device of a type
 * @type: the type of backlight device
 *
 * Look up the first backlight device of the specified type
 *
 * RETURNS:
 *
 * Pointer to backlight device if any was found. Otherwise NULL.
 */
struct backlight_device *backlight_device_get_by_type(enum backlight_type type)
{
        bool found = false;
        struct backlight_device *bd;

        mutex_lock(&backlight_dev_list_mutex);
        list_for_each_entry(bd, &backlight_dev_list, entry) {
                if (bd->props.type == type) {
                        found = true;
                        break;
                }
        }
        mutex_unlock(&backlight_dev_list_mutex);

        return found ? bd : NULL;
}
EXPORT_SYMBOL(backlight_device_get_by_type);

/**
 * backlight_device_get_by_name - Get backlight device by name
 * @name: Device name
 *
 * This function looks up a backlight device by its name. It obtains a reference
 * on the backlight device and it is the caller's responsibility to drop the
 * reference by calling put_device().
 *
 * Returns:
 * A pointer to the backlight device if found, otherwise NULL.
 */
struct backlight_device *backlight_device_get_by_name(const char *name)
{
        struct device *dev;

        dev = class_find_device_by_name(&backlight_class, name);

        return dev ? to_backlight_device(dev) : NULL;
}
EXPORT_SYMBOL(backlight_device_get_by_name);

/* deprecated - use devm_backlight_device_unregister() */
void backlight_device_unregister(struct backlight_device *bd)
{
        if (!bd)
                return;

        mutex_lock(&backlight_dev_list_mutex);
        list_del(&bd->entry);
        mutex_unlock(&backlight_dev_list_mutex);

#ifdef CONFIG_PMAC_BACKLIGHT
        mutex_lock(&pmac_backlight_mutex);
        if (pmac_backlight == bd)
                pmac_backlight = NULL;
        mutex_unlock(&pmac_backlight_mutex);
#endif

        blocking_notifier_call_chain(&backlight_notifier,
                                     BACKLIGHT_UNREGISTERED, bd);

        mutex_lock(&bd->ops_lock);
        bd->ops = NULL;
        mutex_unlock(&bd->ops_lock);

        backlight_unregister_fb(bd);
        device_unregister(&bd->dev);
}
EXPORT_SYMBOL(backlight_device_unregister);

static void devm_backlight_device_release(struct device *dev, void *res)
{
        struct backlight_device *backlight = *(struct backlight_device **)res;

        backlight_device_unregister(backlight);
}

static int devm_backlight_device_match(struct device *dev, void *res,
                                        void *data)
{
        struct backlight_device **r = res;

        return *r == data;
}

/**
 * backlight_register_notifier - get notified of backlight (un)registration
 * @nb: notifier block with the notifier to call on backlight (un)registration
 *
 * Register a notifier to get notified when backlight devices get registered
 * or unregistered.
 *
 * RETURNS:
 *
 * 0 on success, otherwise a negative error code
 */
int backlight_register_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&backlight_notifier, nb);
}
EXPORT_SYMBOL(backlight_register_notifier);

/**
 * backlight_unregister_notifier - unregister a backlight notifier
 * @nb: notifier block to unregister
 *
 * Register a notifier to get notified when backlight devices get registered
 * or unregistered.
 *
 * RETURNS:
 *
 * 0 on success, otherwise a negative error code
 */
int backlight_unregister_notifier(struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&backlight_notifier, nb);
}
EXPORT_SYMBOL(backlight_unregister_notifier);

/**
 * devm_backlight_device_register - register a new backlight device
 * @dev: the device to register
 * @name: the name of the device
 * @parent: a pointer to the parent device (often the same as @dev)
 * @devdata: an optional pointer to be stored for private driver use
 * @ops: the backlight operations structure
 * @props: the backlight properties
 *
 * Creates and registers new backlight device. When a backlight device
 * is registered the configuration must be specified in the @props
 * parameter. See description of &backlight_properties.
 *
 * RETURNS:
 *
 * struct backlight on success, or an ERR_PTR on error
 */
struct backlight_device *devm_backlight_device_register(struct device *dev,
        const char *name, struct device *parent, void *devdata,
        const struct backlight_ops *ops,
        const struct backlight_properties *props)
{
        struct backlight_device **ptr, *backlight;

        ptr = devres_alloc(devm_backlight_device_release, sizeof(*ptr),
                        GFP_KERNEL);
        if (!ptr)
                return ERR_PTR(-ENOMEM);

        backlight = backlight_device_register(name, parent, devdata, ops,
                                                props);
        if (!IS_ERR(backlight)) {
                *ptr = backlight;
                devres_add(dev, ptr);
        } else {
                devres_free(ptr);
        }

        return backlight;
}
EXPORT_SYMBOL(devm_backlight_device_register);

/**
 * devm_backlight_device_unregister - unregister backlight device
 * @dev: the device to unregister
 * @bd: the backlight device to unregister
 *
 * Deallocates a backlight allocated with devm_backlight_device_register().
 * Normally this function will not need to be called and the resource management
 * code will ensure that the resources are freed.
 */
void devm_backlight_device_unregister(struct device *dev,
                                struct backlight_device *bd)
{
        int rc;

        rc = devres_release(dev, devm_backlight_device_release,
                                devm_backlight_device_match, bd);
        WARN_ON(rc);
}
EXPORT_SYMBOL(devm_backlight_device_unregister);

#ifdef CONFIG_OF
static int of_parent_match(struct device *dev, const void *data)
{
        return dev->parent && dev->parent->of_node == data;
}

/**
 * of_find_backlight_by_node() - find backlight device by device-tree node
 * @node: device-tree node of the backlight device
 *
 * Returns a pointer to the backlight device corresponding to the given DT
 * node or NULL if no such backlight device exists or if the device hasn't
 * been probed yet.
 *
 * This function obtains a reference on the backlight device and it is the
 * caller's responsibility to drop the reference by calling put_device() on
 * the backlight device's .dev field.
 */
struct backlight_device *of_find_backlight_by_node(struct device_node *node)
{
        struct device *dev;

        dev = class_find_device(&backlight_class, NULL, node, of_parent_match);

        return dev ? to_backlight_device(dev) : NULL;
}
EXPORT_SYMBOL(of_find_backlight_by_node);
#endif

static struct backlight_device *of_find_backlight(struct device *dev)
{
        struct backlight_device *bd = NULL;
        struct device_node *np;

        if (!dev)
                return NULL;

        if (IS_ENABLED(CONFIG_OF) && dev->of_node) {
                np = of_parse_phandle(dev->of_node, "backlight", 0);
                if (np) {
                        bd = of_find_backlight_by_node(np);
                        of_node_put(np);
                        if (!bd)
                                return ERR_PTR(-EPROBE_DEFER);
                }
        }

        return bd;
}

static void devm_backlight_release(void *data)
{
        struct backlight_device *bd = data;

        put_device(&bd->dev);
}

/**
 * devm_of_find_backlight - find backlight for a device
 * @dev: the device
 *
 * This function looks for a property named 'backlight' on the DT node
 * connected to @dev and looks up the backlight device. The lookup is
 * device managed so the reference to the backlight device is automatically
 * dropped on driver detach.
 *
 * RETURNS:
 *
 * A pointer to the backlight device if found.
 * Error pointer -EPROBE_DEFER if the DT property is set, but no backlight
 * device is found. NULL if there's no backlight property.
 */
struct backlight_device *devm_of_find_backlight(struct device *dev)
{
        struct backlight_device *bd;
        int ret;

        bd = of_find_backlight(dev);
        if (IS_ERR_OR_NULL(bd))
                return bd;
        ret = devm_add_action_or_reset(dev, devm_backlight_release, bd);
        if (ret)
                return ERR_PTR(ret);

        return bd;
}
EXPORT_SYMBOL(devm_of_find_backlight);

static void __exit backlight_class_exit(void)
{
        class_unregister(&backlight_class);
}

static int __init backlight_class_init(void)
{
        int ret;

        ret = class_register(&backlight_class);
        if (ret) {
                pr_warn("Unable to create backlight class; errno = %d\n", ret);
                return ret;
        }

        INIT_LIST_HEAD(&backlight_dev_list);
        mutex_init(&backlight_dev_list_mutex);
        BLOCKING_INIT_NOTIFIER_HEAD(&backlight_notifier);

        return 0;
}

/*
 * if this is compiled into the kernel, we need to ensure that the
 * class is registered before users of the class try to register lcd's
 */
postcore_initcall(backlight_class_init);
module_exit(backlight_class_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Jamey Hicks <jamey.hicks@hp.com>, Andrew Zabolotny <zap@homelink.ru>");
MODULE_DESCRIPTION("Backlight Lowlevel Control Abstraction");
















  105 
  104 













  102 








   10 
   10 













   10 








   12 






















    1 


   11 











































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
// SPDX-License-Identifier: GPL-2.0-only
/*
 * 32bit compatibility wrappers for the input subsystem.
 *
 * Very heavily based on evdev.c - Copyright (c) 1999-2002 Vojtech Pavlik
 */

#include <linux/export.h>
#include <linux/uaccess.h>
#include "input-compat.h"

#ifdef CONFIG_COMPAT

int input_event_from_user(const char __user *buffer,
                          struct input_event *event)
{
        if (in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
                struct input_event_compat compat_event;

                if (copy_from_user(&compat_event, buffer,
                                   sizeof(struct input_event_compat)))
                        return -EFAULT;

                event->input_event_sec = compat_event.sec;
                event->input_event_usec = compat_event.usec;
                event->type = compat_event.type;
                event->code = compat_event.code;
                event->value = compat_event.value;

        } else {
                if (copy_from_user(event, buffer, sizeof(struct input_event)))
                        return -EFAULT;
        }

        return 0;
}

int input_event_to_user(char __user *buffer,
                        const struct input_event *event)
{
        if (in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
                struct input_event_compat compat_event;

                compat_event.sec = event->input_event_sec;
                compat_event.usec = event->input_event_usec;
                compat_event.type = event->type;
                compat_event.code = event->code;
                compat_event.value = event->value;

                if (copy_to_user(buffer, &compat_event,
                                 sizeof(struct input_event_compat)))
                        return -EFAULT;

        } else {
                if (copy_to_user(buffer, event, sizeof(struct input_event)))
                        return -EFAULT;
        }

        return 0;
}

int input_ff_effect_from_user(const char __user *buffer, size_t size,
                              struct ff_effect *effect)
{
        if (in_compat_syscall()) {
                struct ff_effect_compat *compat_effect;

                if (size != sizeof(struct ff_effect_compat))
                        return -EINVAL;

                /*
                 * It so happens that the pointer which needs to be changed
                 * is the last field in the structure, so we can retrieve the
                 * whole thing and replace just the pointer.
                 */
                compat_effect = (struct ff_effect_compat *)effect;

                if (copy_from_user(compat_effect, buffer,
                                   sizeof(struct ff_effect_compat)))
                        return -EFAULT;

                if (compat_effect->type == FF_PERIODIC &&
                    compat_effect->u.periodic.waveform == FF_CUSTOM)
                        effect->u.periodic.custom_data =
                                compat_ptr(compat_effect->u.periodic.custom_data);
        } else {
                if (size != sizeof(struct ff_effect))
                        return -EINVAL;

                if (copy_from_user(effect, buffer, sizeof(struct ff_effect)))
                        return -EFAULT;
        }

        return 0;
}

#else

int input_event_from_user(const char __user *buffer,
                         struct input_event *event)
{
        if (copy_from_user(event, buffer, sizeof(struct input_event)))
                return -EFAULT;

        return 0;
}

int input_event_to_user(char __user *buffer,
                        const struct input_event *event)
{
        if (copy_to_user(buffer, event, sizeof(struct input_event)))
                return -EFAULT;

        return 0;
}

int input_ff_effect_from_user(const char __user *buffer, size_t size,
                              struct ff_effect *effect)
{
        if (size != sizeof(struct ff_effect))
                return -EINVAL;

        if (copy_from_user(effect, buffer, sizeof(struct ff_effect)))
                return -EFAULT;

        return 0;
}

#endif /* CONFIG_COMPAT */

EXPORT_SYMBOL_GPL(input_event_from_user);
EXPORT_SYMBOL_GPL(input_event_to_user);
EXPORT_SYMBOL_GPL(input_ff_effect_from_user);













































    1 


    1 




    1 











    5 




    5 



    5 





    2 




    5 








    8 





    8 



    2 



    4 

    1 

    4 

    3 



    2 
    2 


    2 







    2 






   13 



   13 


   13 

   13 

   13 


























    7 










    3 

    3 














    2 
    2 







    2 





   18 




   18 



    2 

    1 



   10 
   10 






   10 



    2 



    3 

    3 









































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
// SPDX-License-Identifier: GPL-2.0-only
/*
 *
 * general timer device for using in ISDN stacks
 *
 * Author        Karsten Keil <kkeil@novell.com>
 *
 * Copyright 2008  by Karsten Keil <kkeil@novell.com>
 */

#include <linux/poll.h>
#include <linux/vmalloc.h>
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/miscdevice.h>
#include <linux/module.h>
#include <linux/mISDNif.h>
#include <linux/mutex.h>
#include <linux/sched/signal.h>

#include "core.h"

static DEFINE_MUTEX(mISDN_mutex);
static u_int        *debug;


struct mISDNtimerdev {
        int                        next_id;
        struct list_head        pending;
        struct list_head        expired;
        wait_queue_head_t        wait;
        u_int                        work;
        spinlock_t                lock; /* protect lists */
};

struct mISDNtimer {
        struct list_head        list;
        struct  mISDNtimerdev        *dev;
        struct timer_list        tl;
        int                        id;
};

static int
mISDN_open(struct inode *ino, struct file *filep)
{
        struct mISDNtimerdev        *dev;

        if (*debug & DEBUG_TIMER)
                printk(KERN_DEBUG "%s(%p,%p)\n", __func__, ino, filep);
        dev = kmalloc(sizeof(struct mISDNtimerdev) , GFP_KERNEL);
        if (!dev)
                return -ENOMEM;
        dev->next_id = 1;
        INIT_LIST_HEAD(&dev->pending);
        INIT_LIST_HEAD(&dev->expired);
        spin_lock_init(&dev->lock);
        dev->work = 0;
        init_waitqueue_head(&dev->wait);
        filep->private_data = dev;
        return nonseekable_open(ino, filep);
}

static int
mISDN_close(struct inode *ino, struct file *filep)
{
        struct mISDNtimerdev        *dev = filep->private_data;
        struct list_head        *list = &dev->pending;
        struct mISDNtimer        *timer, *next;

        if (*debug & DEBUG_TIMER)
                printk(KERN_DEBUG "%s(%p,%p)\n", __func__, ino, filep);

        spin_lock_irq(&dev->lock);
        while (!list_empty(list)) {
                timer = list_first_entry(list, struct mISDNtimer, list);
                spin_unlock_irq(&dev->lock);
                timer_shutdown_sync(&timer->tl);
                spin_lock_irq(&dev->lock);
                /* it might have been moved to ->expired */
                list_del(&timer->list);
                kfree(timer);
        }
        spin_unlock_irq(&dev->lock);

        list_for_each_entry_safe(timer, next, &dev->expired, list) {
                kfree(timer);
        }
        kfree(dev);
        return 0;
}

static ssize_t
mISDN_read(struct file *filep, char __user *buf, size_t count, loff_t *off)
{
        struct mISDNtimerdev        *dev = filep->private_data;
        struct list_head *list = &dev->expired;
        struct mISDNtimer        *timer;
        int        ret = 0;

        if (*debug & DEBUG_TIMER)
                printk(KERN_DEBUG "%s(%p, %p, %d, %p)\n", __func__,
                       filep, buf, (int)count, off);

        if (count < sizeof(int))
                return -ENOSPC;

        spin_lock_irq(&dev->lock);
        while (list_empty(list) && (dev->work == 0)) {
                spin_unlock_irq(&dev->lock);
                if (filep->f_flags & O_NONBLOCK)
                        return -EAGAIN;
                wait_event_interruptible(dev->wait, (dev->work ||
                                                     !list_empty(list)));
                if (signal_pending(current))
                        return -ERESTARTSYS;
                spin_lock_irq(&dev->lock);
        }
        if (dev->work)
                dev->work = 0;
        if (!list_empty(list)) {
                timer = list_first_entry(list, struct mISDNtimer, list);
                list_del(&timer->list);
                spin_unlock_irq(&dev->lock);
                if (put_user(timer->id, (int __user *)buf))
                        ret = -EFAULT;
                else
                        ret = sizeof(int);
                kfree(timer);
        } else {
                spin_unlock_irq(&dev->lock);
        }
        return ret;
}

static __poll_t
mISDN_poll(struct file *filep, poll_table *wait)
{
        struct mISDNtimerdev        *dev = filep->private_data;
        __poll_t                mask = EPOLLERR;

        if (*debug & DEBUG_TIMER)
                printk(KERN_DEBUG "%s(%p, %p)\n", __func__, filep, wait);
        if (dev) {
                poll_wait(filep, &dev->wait, wait);
                mask = 0;
                if (dev->work || !list_empty(&dev->expired))
                        mask |= (EPOLLIN | EPOLLRDNORM);
                if (*debug & DEBUG_TIMER)
                        printk(KERN_DEBUG "%s work(%d) empty(%d)\n", __func__,
                               dev->work, list_empty(&dev->expired));
        }
        return mask;
}

static void
dev_expire_timer(struct timer_list *t)
{
        struct mISDNtimer *timer = from_timer(timer, t, tl);
        u_long                        flags;

        spin_lock_irqsave(&timer->dev->lock, flags);
        if (timer->id >= 0)
                list_move_tail(&timer->list, &timer->dev->expired);
        wake_up_interruptible(&timer->dev->wait);
        spin_unlock_irqrestore(&timer->dev->lock, flags);
}

static int
misdn_add_timer(struct mISDNtimerdev *dev, int timeout)
{
        int                        id;
        struct mISDNtimer        *timer;

        if (!timeout) {
                dev->work = 1;
                wake_up_interruptible(&dev->wait);
                id = 0;
        } else {
                timer = kzalloc(sizeof(struct mISDNtimer), GFP_KERNEL);
                if (!timer)
                        return -ENOMEM;
                timer->dev = dev;
                timer_setup(&timer->tl, dev_expire_timer, 0);
                spin_lock_irq(&dev->lock);
                id = timer->id = dev->next_id++;
                if (dev->next_id < 0)
                        dev->next_id = 1;
                list_add_tail(&timer->list, &dev->pending);
                timer->tl.expires = jiffies + ((HZ * (u_long)timeout) / 1000);
                add_timer(&timer->tl);
                spin_unlock_irq(&dev->lock);
        }
        return id;
}

static int
misdn_del_timer(struct mISDNtimerdev *dev, int id)
{
        struct mISDNtimer        *timer;

        spin_lock_irq(&dev->lock);
        list_for_each_entry(timer, &dev->pending, list) {
                if (timer->id == id) {
                        list_del_init(&timer->list);
                        timer->id = -1;
                        spin_unlock_irq(&dev->lock);
                        timer_shutdown_sync(&timer->tl);
                        kfree(timer);
                        return id;
                }
        }
        spin_unlock_irq(&dev->lock);
        return 0;
}

static long
mISDN_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
{
        struct mISDNtimerdev        *dev = filep->private_data;
        int                        id, tout, ret = 0;


        if (*debug & DEBUG_TIMER)
                printk(KERN_DEBUG "%s(%p, %x, %lx)\n", __func__,
                       filep, cmd, arg);
        mutex_lock(&mISDN_mutex);
        switch (cmd) {
        case IMADDTIMER:
                if (get_user(tout, (int __user *)arg)) {
                        ret = -EFAULT;
                        break;
                }
                id = misdn_add_timer(dev, tout);
                if (*debug & DEBUG_TIMER)
                        printk(KERN_DEBUG "%s add %d id %d\n", __func__,
                               tout, id);
                if (id < 0) {
                        ret = id;
                        break;
                }
                if (put_user(id, (int __user *)arg))
                        ret = -EFAULT;
                break;
        case IMDELTIMER:
                if (get_user(id, (int __user *)arg)) {
                        ret = -EFAULT;
                        break;
                }
                if (*debug & DEBUG_TIMER)
                        printk(KERN_DEBUG "%s del id %d\n", __func__, id);
                id = misdn_del_timer(dev, id);
                if (put_user(id, (int __user *)arg))
                        ret = -EFAULT;
                break;
        default:
                ret = -EINVAL;
        }
        mutex_unlock(&mISDN_mutex);
        return ret;
}

static const struct file_operations mISDN_fops = {
        .owner                = THIS_MODULE,
        .read                = mISDN_read,
        .poll                = mISDN_poll,
        .unlocked_ioctl        = mISDN_ioctl,
        .open                = mISDN_open,
        .release        = mISDN_close,
};

static struct miscdevice mISDNtimer = {
        .minor        = MISC_DYNAMIC_MINOR,
        .name        = "mISDNtimer",
        .fops        = &mISDN_fops,
};

int
mISDN_inittimer(u_int *deb)
{
        int        err;

        debug = deb;
        err = misc_register(&mISDNtimer);
        if (err)
                printk(KERN_WARNING "mISDN: Could not register timer device\n");
        return err;
}

void mISDN_timer_cleanup(void)
{
        misc_deregister(&mISDNtimer);
}








































































































































































   87 

   42 













































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __SHMEM_FS_H
#define __SHMEM_FS_H

#include <linux/file.h>
#include <linux/swap.h>
#include <linux/mempolicy.h>
#include <linux/pagemap.h>
#include <linux/percpu_counter.h>
#include <linux/xattr.h>
#include <linux/fs_parser.h>
#include <linux/userfaultfd_k.h>

/* inode in-kernel data */

#ifdef CONFIG_TMPFS_QUOTA
#define SHMEM_MAXQUOTAS 2
#endif

struct shmem_inode_info {
        spinlock_t                lock;
        unsigned int                seals;                /* shmem seals */
        unsigned long                flags;
        unsigned long                alloced;        /* data pages alloced to file */
        unsigned long                swapped;        /* subtotal assigned to swap */
        union {
            struct offset_ctx        dir_offsets;        /* stable directory offsets */
            struct {
                struct list_head shrinklist;        /* shrinkable hpage inodes */
                struct list_head swaplist;        /* chain of maybes on swap */
            };
        };
        struct timespec64        i_crtime;        /* file creation time */
        struct shared_policy        policy;                /* NUMA memory alloc policy */
        struct simple_xattrs        xattrs;                /* list of xattrs */
        pgoff_t                        fallocend;        /* highest fallocate endindex */
        unsigned int                fsflags;        /* for FS_IOC_[SG]ETFLAGS */
        atomic_t                stop_eviction;        /* hold when working on inode */
#ifdef CONFIG_TMPFS_QUOTA
        struct dquot __rcu        *i_dquot[MAXQUOTAS];
#endif
        struct inode                vfs_inode;
};

#define SHMEM_FL_USER_VISIBLE                FS_FL_USER_VISIBLE
#define SHMEM_FL_USER_MODIFIABLE \
        (FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NODUMP_FL | FS_NOATIME_FL)
#define SHMEM_FL_INHERITED                (FS_NODUMP_FL | FS_NOATIME_FL)

struct shmem_quota_limits {
        qsize_t usrquota_bhardlimit; /* Default user quota block hard limit */
        qsize_t usrquota_ihardlimit; /* Default user quota inode hard limit */
        qsize_t grpquota_bhardlimit; /* Default group quota block hard limit */
        qsize_t grpquota_ihardlimit; /* Default group quota inode hard limit */
};

struct shmem_sb_info {
        unsigned long max_blocks;   /* How many blocks are allowed */
        struct percpu_counter used_blocks;  /* How many are allocated */
        unsigned long max_inodes;   /* How many inodes are allowed */
        unsigned long free_ispace;  /* How much ispace left for allocation */
        raw_spinlock_t stat_lock;   /* Serialize shmem_sb_info changes */
        umode_t mode;                    /* Mount mode for root directory */
        unsigned char huge;            /* Whether to try for hugepages */
        kuid_t uid;                    /* Mount uid for root directory */
        kgid_t gid;                    /* Mount gid for root directory */
        bool full_inums;            /* If i_ino should be uint or ino_t */
        bool noswap;                    /* ignores VM reclaim / swap requests */
        ino_t next_ino;                    /* The next per-sb inode number to use */
        ino_t __percpu *ino_batch;  /* The next per-cpu inode number to use */
        struct mempolicy *mpol;     /* default memory policy for mappings */
        spinlock_t shrinklist_lock;   /* Protects shrinklist */
        struct list_head shrinklist;  /* List of shinkable inodes */
        unsigned long shrinklist_len; /* Length of shrinklist */
        struct shmem_quota_limits qlimits; /* Default quota limits */
};

static inline struct shmem_inode_info *SHMEM_I(struct inode *inode)
{
        return container_of(inode, struct shmem_inode_info, vfs_inode);
}

/*
 * Functions in mm/shmem.c called directly from elsewhere:
 */
extern const struct fs_parameter_spec shmem_fs_parameters[];
extern void shmem_init(void);
extern int shmem_init_fs_context(struct fs_context *fc);
extern struct file *shmem_file_setup(const char *name,
                                        loff_t size, unsigned long flags);
extern struct file *shmem_kernel_file_setup(const char *name, loff_t size,
                                            unsigned long flags);
extern struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt,
                const char *name, loff_t size, unsigned long flags);
extern int shmem_zero_setup(struct vm_area_struct *);
extern unsigned long shmem_get_unmapped_area(struct file *, unsigned long addr,
                unsigned long len, unsigned long pgoff, unsigned long flags);
extern int shmem_lock(struct file *file, int lock, struct ucounts *ucounts);
#ifdef CONFIG_SHMEM
bool shmem_mapping(struct address_space *mapping);
#else
static inline bool shmem_mapping(struct address_space *mapping)
{
        return false;
}
#endif /* CONFIG_SHMEM */
extern void shmem_unlock_mapping(struct address_space *mapping);
extern struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
                                        pgoff_t index, gfp_t gfp_mask);
extern void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end);
int shmem_unuse(unsigned int type);

#ifdef CONFIG_TRANSPARENT_HUGEPAGE
unsigned long shmem_allowable_huge_orders(struct inode *inode,
                                struct vm_area_struct *vma, pgoff_t index,
                                loff_t write_end, bool shmem_huge_force);
#else
static inline unsigned long shmem_allowable_huge_orders(struct inode *inode,
                                struct vm_area_struct *vma, pgoff_t index,
                                loff_t write_end, bool shmem_huge_force)
{
        return 0;
}
#endif

#ifdef CONFIG_SHMEM
extern unsigned long shmem_swap_usage(struct vm_area_struct *vma);
#else
static inline unsigned long shmem_swap_usage(struct vm_area_struct *vma)
{
        return 0;
}
#endif
extern unsigned long shmem_partial_swap_usage(struct address_space *mapping,
                                                pgoff_t start, pgoff_t end);

/* Flag allocation requirements to shmem_get_folio */
enum sgp_type {
        SGP_READ,        /* don't exceed i_size, don't allocate page */
        SGP_NOALLOC,        /* similar, but fail on hole or use fallocated page */
        SGP_CACHE,        /* don't exceed i_size, may allocate page */
        SGP_WRITE,        /* may exceed i_size, may allocate !Uptodate page */
        SGP_FALLOC,        /* like SGP_WRITE, but make existing page Uptodate */
};

int shmem_get_folio(struct inode *inode, pgoff_t index, loff_t write_end,
                struct folio **foliop, enum sgp_type sgp);
struct folio *shmem_read_folio_gfp(struct address_space *mapping,
                pgoff_t index, gfp_t gfp);

static inline struct folio *shmem_read_folio(struct address_space *mapping,
                pgoff_t index)
{
        return shmem_read_folio_gfp(mapping, index, mapping_gfp_mask(mapping));
}

static inline struct page *shmem_read_mapping_page(
                                struct address_space *mapping, pgoff_t index)
{
        return shmem_read_mapping_page_gfp(mapping, index,
                                        mapping_gfp_mask(mapping));
}

static inline bool shmem_file(struct file *file)
{
        if (!IS_ENABLED(CONFIG_SHMEM))
                return false;
        if (!file || !file->f_mapping)
                return false;
        return shmem_mapping(file->f_mapping);
}

/*
 * If fallocate(FALLOC_FL_KEEP_SIZE) has been used, there may be pages
 * beyond i_size's notion of EOF, which fallocate has committed to reserving:
 * which split_huge_page() must therefore not delete.  This use of a single
 * "fallocend" per inode errs on the side of not deleting a reservation when
 * in doubt: there are plenty of cases when it preserves unreserved pages.
 */
static inline pgoff_t shmem_fallocend(struct inode *inode, pgoff_t eof)
{
        return max(eof, SHMEM_I(inode)->fallocend);
}

extern bool shmem_charge(struct inode *inode, long pages);
extern void shmem_uncharge(struct inode *inode, long pages);

#ifdef CONFIG_USERFAULTFD
#ifdef CONFIG_SHMEM
extern int shmem_mfill_atomic_pte(pmd_t *dst_pmd,
                                  struct vm_area_struct *dst_vma,
                                  unsigned long dst_addr,
                                  unsigned long src_addr,
                                  uffd_flags_t flags,
                                  struct folio **foliop);
#else /* !CONFIG_SHMEM */
#define shmem_mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, \
                               src_addr, flags, foliop) ({ BUG(); 0; })
#endif /* CONFIG_SHMEM */
#endif /* CONFIG_USERFAULTFD */

/*
 * Used space is stored as unsigned 64-bit value in bytes but
 * quota core supports only signed 64-bit values so use that
 * as a limit
 */
#define SHMEM_QUOTA_MAX_SPC_LIMIT 0x7fffffffffffffffLL /* 2^63-1 */
#define SHMEM_QUOTA_MAX_INO_LIMIT 0x7fffffffffffffffLL

#ifdef CONFIG_TMPFS_QUOTA
extern const struct dquot_operations shmem_quota_operations;
extern struct quota_format_type shmem_quota_format;
#endif /* CONFIG_TMPFS_QUOTA */

#endif

























































































































   25 


    9 



   15 
   15 
   15 
   15 
   15 
   15 
   15 
   15 
   15 
   15 


    1 




























































































































































































































































    3 
    3 










   21 


    2 


   14 


    1 


    1 


    1 



    1 


    1 


















    1 



    3 

    3 





    2 

    2 



    1 








   43 


















    2 


    1 


   15 






    1 






    1 






    1 







    1 






    1 




    9 



    1 


    2 


    2 


    1 



    1 



    1 









    1 


    1 



    1 



    1 



    1 



    1 



    1 






    1 



    1 







    1 









    1 
    1 








    2 
    1 



    1 








    1 
    1 

    2 


    2 


























































































































































































































































































































































































































































































































   17 







    1 


    1 


    1 


    1 


    1 


    9 

    9 

    9 

    9 

    9 



    1 






    1 




























































































    3 






    2 

    1 

















































    1 


































    1 






























































    6 


    1 

    1 










    1 



    1 







    1 



    1 
























    1 
    1 



















































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
// SPDX-License-Identifier: GPL-2.0-only
/*
 * vivid-ctrls.c - control support functions.
 *
 * Copyright 2014 Cisco Systems, Inc. and/or its affiliates. All rights reserved.
 */

#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/videodev2.h>
#include <media/v4l2-event.h>
#include <media/v4l2-common.h>

#include "vivid-core.h"
#include "vivid-vid-cap.h"
#include "vivid-vid-out.h"
#include "vivid-vid-common.h"
#include "vivid-radio-common.h"
#include "vivid-osd.h"
#include "vivid-ctrls.h"

#define VIVID_CID_CUSTOM_BASE                (V4L2_CID_USER_BASE | 0xf000)
#define VIVID_CID_BUTTON                (VIVID_CID_CUSTOM_BASE + 0)
#define VIVID_CID_BOOLEAN                (VIVID_CID_CUSTOM_BASE + 1)
#define VIVID_CID_INTEGER                (VIVID_CID_CUSTOM_BASE + 2)
#define VIVID_CID_INTEGER64                (VIVID_CID_CUSTOM_BASE + 3)
#define VIVID_CID_MENU                        (VIVID_CID_CUSTOM_BASE + 4)
#define VIVID_CID_STRING                (VIVID_CID_CUSTOM_BASE + 5)
#define VIVID_CID_BITMASK                (VIVID_CID_CUSTOM_BASE + 6)
#define VIVID_CID_INTMENU                (VIVID_CID_CUSTOM_BASE + 7)
#define VIVID_CID_U32_ARRAY                (VIVID_CID_CUSTOM_BASE + 8)
#define VIVID_CID_U16_MATRIX                (VIVID_CID_CUSTOM_BASE + 9)
#define VIVID_CID_U8_4D_ARRAY                (VIVID_CID_CUSTOM_BASE + 10)
#define VIVID_CID_AREA                        (VIVID_CID_CUSTOM_BASE + 11)
#define VIVID_CID_RO_INTEGER                (VIVID_CID_CUSTOM_BASE + 12)
#define VIVID_CID_U32_DYN_ARRAY                (VIVID_CID_CUSTOM_BASE + 13)
#define VIVID_CID_U8_PIXEL_ARRAY        (VIVID_CID_CUSTOM_BASE + 14)
#define VIVID_CID_S32_ARRAY                (VIVID_CID_CUSTOM_BASE + 15)
#define VIVID_CID_S64_ARRAY                (VIVID_CID_CUSTOM_BASE + 16)

#define VIVID_CID_VIVID_BASE                (0x00f00000 | 0xf000)
#define VIVID_CID_VIVID_CLASS                (0x00f00000 | 1)
#define VIVID_CID_TEST_PATTERN                (VIVID_CID_VIVID_BASE + 0)
#define VIVID_CID_OSD_TEXT_MODE                (VIVID_CID_VIVID_BASE + 1)
#define VIVID_CID_HOR_MOVEMENT                (VIVID_CID_VIVID_BASE + 2)
#define VIVID_CID_VERT_MOVEMENT                (VIVID_CID_VIVID_BASE + 3)
#define VIVID_CID_SHOW_BORDER                (VIVID_CID_VIVID_BASE + 4)
#define VIVID_CID_SHOW_SQUARE                (VIVID_CID_VIVID_BASE + 5)
#define VIVID_CID_INSERT_SAV                (VIVID_CID_VIVID_BASE + 6)
#define VIVID_CID_INSERT_EAV                (VIVID_CID_VIVID_BASE + 7)
#define VIVID_CID_VBI_CAP_INTERLACED        (VIVID_CID_VIVID_BASE + 8)
#define VIVID_CID_INSERT_HDMI_VIDEO_GUARD_BAND (VIVID_CID_VIVID_BASE + 9)

#define VIVID_CID_HFLIP                        (VIVID_CID_VIVID_BASE + 20)
#define VIVID_CID_VFLIP                        (VIVID_CID_VIVID_BASE + 21)
#define VIVID_CID_STD_ASPECT_RATIO        (VIVID_CID_VIVID_BASE + 22)
#define VIVID_CID_DV_TIMINGS_ASPECT_RATIO        (VIVID_CID_VIVID_BASE + 23)
#define VIVID_CID_TSTAMP_SRC                (VIVID_CID_VIVID_BASE + 24)
#define VIVID_CID_COLORSPACE                (VIVID_CID_VIVID_BASE + 25)
#define VIVID_CID_XFER_FUNC                (VIVID_CID_VIVID_BASE + 26)
#define VIVID_CID_YCBCR_ENC                (VIVID_CID_VIVID_BASE + 27)
#define VIVID_CID_QUANTIZATION                (VIVID_CID_VIVID_BASE + 28)
#define VIVID_CID_LIMITED_RGB_RANGE        (VIVID_CID_VIVID_BASE + 29)
#define VIVID_CID_ALPHA_MODE                (VIVID_CID_VIVID_BASE + 30)
#define VIVID_CID_HAS_CROP_CAP                (VIVID_CID_VIVID_BASE + 31)
#define VIVID_CID_HAS_COMPOSE_CAP        (VIVID_CID_VIVID_BASE + 32)
#define VIVID_CID_HAS_SCALER_CAP        (VIVID_CID_VIVID_BASE + 33)
#define VIVID_CID_HAS_CROP_OUT                (VIVID_CID_VIVID_BASE + 34)
#define VIVID_CID_HAS_COMPOSE_OUT        (VIVID_CID_VIVID_BASE + 35)
#define VIVID_CID_HAS_SCALER_OUT        (VIVID_CID_VIVID_BASE + 36)
#define VIVID_CID_SEQ_WRAP                (VIVID_CID_VIVID_BASE + 38)
#define VIVID_CID_TIME_WRAP                (VIVID_CID_VIVID_BASE + 39)
#define VIVID_CID_MAX_EDID_BLOCKS        (VIVID_CID_VIVID_BASE + 40)
#define VIVID_CID_PERCENTAGE_FILL        (VIVID_CID_VIVID_BASE + 41)
#define VIVID_CID_REDUCED_FPS                (VIVID_CID_VIVID_BASE + 42)
#define VIVID_CID_HSV_ENC                (VIVID_CID_VIVID_BASE + 43)

#define VIVID_CID_STD_SIGNAL_MODE        (VIVID_CID_VIVID_BASE + 60)
#define VIVID_CID_STANDARD                (VIVID_CID_VIVID_BASE + 61)
#define VIVID_CID_DV_TIMINGS_SIGNAL_MODE        (VIVID_CID_VIVID_BASE + 62)
#define VIVID_CID_DV_TIMINGS                (VIVID_CID_VIVID_BASE + 63)
#define VIVID_CID_PERC_DROPPED                (VIVID_CID_VIVID_BASE + 64)
#define VIVID_CID_DISCONNECT                (VIVID_CID_VIVID_BASE + 65)
#define VIVID_CID_DQBUF_ERROR                (VIVID_CID_VIVID_BASE + 66)
#define VIVID_CID_QUEUE_SETUP_ERROR        (VIVID_CID_VIVID_BASE + 67)
#define VIVID_CID_BUF_PREPARE_ERROR        (VIVID_CID_VIVID_BASE + 68)
#define VIVID_CID_START_STR_ERROR        (VIVID_CID_VIVID_BASE + 69)
#define VIVID_CID_QUEUE_ERROR                (VIVID_CID_VIVID_BASE + 70)
#define VIVID_CID_CLEAR_FB                (VIVID_CID_VIVID_BASE + 71)
#define VIVID_CID_REQ_VALIDATE_ERROR        (VIVID_CID_VIVID_BASE + 72)

#define VIVID_CID_RADIO_SEEK_MODE        (VIVID_CID_VIVID_BASE + 90)
#define VIVID_CID_RADIO_SEEK_PROG_LIM        (VIVID_CID_VIVID_BASE + 91)
#define VIVID_CID_RADIO_RX_RDS_RBDS        (VIVID_CID_VIVID_BASE + 92)
#define VIVID_CID_RADIO_RX_RDS_BLOCKIO        (VIVID_CID_VIVID_BASE + 93)

#define VIVID_CID_RADIO_TX_RDS_BLOCKIO        (VIVID_CID_VIVID_BASE + 94)

#define VIVID_CID_SDR_CAP_FM_DEVIATION        (VIVID_CID_VIVID_BASE + 110)

#define VIVID_CID_META_CAP_GENERATE_PTS        (VIVID_CID_VIVID_BASE + 111)
#define VIVID_CID_META_CAP_GENERATE_SCR        (VIVID_CID_VIVID_BASE + 112)

/* HDMI inputs are in the range 0-14. The next available CID is VIVID_CID_VIVID_BASE + 128 */
#define VIVID_CID_HDMI_IS_CONNECTED_TO_OUTPUT(input) (VIVID_CID_VIVID_BASE + 113 + (input))

/* S-Video inputs are in the range 0-15. The next available CID is VIVID_CID_VIVID_BASE + 144 */
#define VIVID_CID_SVID_IS_CONNECTED_TO_OUTPUT(input) (VIVID_CID_VIVID_BASE + 128 + (input))

/* General User Controls */

static void vivid_unregister_dev(bool valid, struct video_device *vdev)
{
        if (!valid)
                return;
        clear_bit(V4L2_FL_REGISTERED, &vdev->flags);
        v4l2_event_wake_all(vdev);
}

static int vivid_user_gen_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev, ctrl_hdl_user_gen);

        switch (ctrl->id) {
        case VIVID_CID_DISCONNECT:
                v4l2_info(&dev->v4l2_dev, "disconnect\n");
                dev->disconnect_error = true;
                vivid_unregister_dev(dev->has_vid_cap, &dev->vid_cap_dev);
                vivid_unregister_dev(dev->has_vid_out, &dev->vid_out_dev);
                vivid_unregister_dev(dev->has_vbi_cap, &dev->vbi_cap_dev);
                vivid_unregister_dev(dev->has_vbi_out, &dev->vbi_out_dev);
                vivid_unregister_dev(dev->has_radio_rx, &dev->radio_rx_dev);
                vivid_unregister_dev(dev->has_radio_tx, &dev->radio_tx_dev);
                vivid_unregister_dev(dev->has_sdr_cap, &dev->sdr_cap_dev);
                vivid_unregister_dev(dev->has_meta_cap, &dev->meta_cap_dev);
                vivid_unregister_dev(dev->has_meta_out, &dev->meta_out_dev);
                vivid_unregister_dev(dev->has_touch_cap, &dev->touch_cap_dev);
                break;
        case VIVID_CID_BUTTON:
                dev->button_pressed = 30;
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_user_gen_ctrl_ops = {
        .s_ctrl = vivid_user_gen_s_ctrl,
};

static const struct v4l2_ctrl_config vivid_ctrl_button = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_BUTTON,
        .name = "Button",
        .type = V4L2_CTRL_TYPE_BUTTON,
};

static const struct v4l2_ctrl_config vivid_ctrl_boolean = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_BOOLEAN,
        .name = "Boolean",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .min = 0,
        .max = 1,
        .step = 1,
        .def = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_int32 = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_INTEGER,
        .name = "Integer 32 Bits",
        .type = V4L2_CTRL_TYPE_INTEGER,
        .min = 0xffffffff80000000ULL,
        .max = 0x7fffffff,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_int64 = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_INTEGER64,
        .name = "Integer 64 Bits",
        .type = V4L2_CTRL_TYPE_INTEGER64,
        .min = 0x8000000000000000ULL,
        .max = 0x7fffffffffffffffLL,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_u32_array = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_U32_ARRAY,
        .name = "U32 1 Element Array",
        .type = V4L2_CTRL_TYPE_U32,
        .def = 0x18,
        .min = 0x10,
        .max = 0x20000,
        .step = 1,
        .dims = { 1 },
};

static const struct v4l2_ctrl_config vivid_ctrl_u32_dyn_array = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_U32_DYN_ARRAY,
        .name = "U32 Dynamic Array",
        .type = V4L2_CTRL_TYPE_U32,
        .flags = V4L2_CTRL_FLAG_DYNAMIC_ARRAY,
        .def = 50,
        .min = 10,
        .max = 90,
        .step = 1,
        .dims = { 100 },
};

static const struct v4l2_ctrl_config vivid_ctrl_u16_matrix = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_U16_MATRIX,
        .name = "U16 8x16 Matrix",
        .type = V4L2_CTRL_TYPE_U16,
        .def = 0x18,
        .min = 0x10,
        .max = 0x2000,
        .step = 1,
        .dims = { 8, 16 },
};

static const struct v4l2_ctrl_config vivid_ctrl_u8_4d_array = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_U8_4D_ARRAY,
        .name = "U8 2x3x4x5 Array",
        .type = V4L2_CTRL_TYPE_U8,
        .def = 0x18,
        .min = 0x10,
        .max = 0x20,
        .step = 1,
        .dims = { 2, 3, 4, 5 },
};

static const struct v4l2_ctrl_config vivid_ctrl_u8_pixel_array = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_U8_PIXEL_ARRAY,
        .name = "U8 Pixel Array",
        .type = V4L2_CTRL_TYPE_U8,
        .def = 0x80,
        .min = 0x00,
        .max = 0xff,
        .step = 1,
        .dims = { 640 / PIXEL_ARRAY_DIV, 360 / PIXEL_ARRAY_DIV },
};

static const struct v4l2_ctrl_config vivid_ctrl_s32_array = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_S32_ARRAY,
        .name = "S32 2 Element Array",
        .type = V4L2_CTRL_TYPE_INTEGER,
        .def = 2,
        .min = -10,
        .max = 10,
        .step = 1,
        .dims = { 2 },
};

static const struct v4l2_ctrl_config vivid_ctrl_s64_array = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_S64_ARRAY,
        .name = "S64 5 Element Array",
        .type = V4L2_CTRL_TYPE_INTEGER64,
        .def = 4,
        .min = -10,
        .max = 10,
        .step = 1,
        .dims = { 5 },
};

static const char * const vivid_ctrl_menu_strings[] = {
        "Menu Item 0 (Skipped)",
        "Menu Item 1",
        "Menu Item 2 (Skipped)",
        "Menu Item 3",
        "Menu Item 4",
        "Menu Item 5 (Skipped)",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_menu = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_MENU,
        .name = "Menu",
        .type = V4L2_CTRL_TYPE_MENU,
        .min = 1,
        .max = 4,
        .def = 3,
        .menu_skip_mask = 0x04,
        .qmenu = vivid_ctrl_menu_strings,
};

static const struct v4l2_ctrl_config vivid_ctrl_string = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_STRING,
        .name = "String",
        .type = V4L2_CTRL_TYPE_STRING,
        .min = 2,
        .max = 4,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_bitmask = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_BITMASK,
        .name = "Bitmask",
        .type = V4L2_CTRL_TYPE_BITMASK,
        .def = 0x80002000,
        .min = 0,
        .max = 0x80402010,
        .step = 0,
};

static const s64 vivid_ctrl_int_menu_values[] = {
        1, 1, 2, 3, 5, 8, 13, 21, 42,
};

static const struct v4l2_ctrl_config vivid_ctrl_int_menu = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_INTMENU,
        .name = "Integer Menu",
        .type = V4L2_CTRL_TYPE_INTEGER_MENU,
        .min = 1,
        .max = 8,
        .def = 4,
        .menu_skip_mask = 0x02,
        .qmenu_int = vivid_ctrl_int_menu_values,
};

static const struct v4l2_ctrl_config vivid_ctrl_disconnect = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_DISCONNECT,
        .name = "Disconnect",
        .type = V4L2_CTRL_TYPE_BUTTON,
};

static const struct v4l2_area area = {
        .width = 1000,
        .height = 2000,
};

static const struct v4l2_ctrl_config vivid_ctrl_area = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_AREA,
        .name = "Area",
        .type = V4L2_CTRL_TYPE_AREA,
        .p_def.p_const = &area,
};

static const struct v4l2_ctrl_config vivid_ctrl_ro_int32 = {
        .ops = &vivid_user_gen_ctrl_ops,
        .id = VIVID_CID_RO_INTEGER,
        .name = "Read-Only Integer 32 Bits",
        .type = V4L2_CTRL_TYPE_INTEGER,
        .flags = V4L2_CTRL_FLAG_READ_ONLY,
        .min = 0,
        .max = 255,
        .step = 1,
};

/* Framebuffer Controls */

static int vivid_fb_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler,
                                             struct vivid_dev, ctrl_hdl_fb);

        switch (ctrl->id) {
        case VIVID_CID_CLEAR_FB:
                vivid_clear_fb(dev);
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_fb_ctrl_ops = {
        .s_ctrl = vivid_fb_s_ctrl,
};

static const struct v4l2_ctrl_config vivid_ctrl_clear_fb = {
        .ops = &vivid_fb_ctrl_ops,
        .id = VIVID_CID_CLEAR_FB,
        .name = "Clear Framebuffer",
        .type = V4L2_CTRL_TYPE_BUTTON,
};


/* Video User Controls */

static int vivid_user_vid_g_volatile_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev, ctrl_hdl_user_vid);

        switch (ctrl->id) {
        case V4L2_CID_AUTOGAIN:
                dev->gain->val = (jiffies_to_msecs(jiffies) / 1000) & 0xff;
                break;
        }
        return 0;
}

static int vivid_user_vid_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev, ctrl_hdl_user_vid);

        switch (ctrl->id) {
        case V4L2_CID_BRIGHTNESS:
                dev->input_brightness[dev->input] = ctrl->val - dev->input * 128;
                tpg_s_brightness(&dev->tpg, dev->input_brightness[dev->input]);
                break;
        case V4L2_CID_CONTRAST:
                tpg_s_contrast(&dev->tpg, ctrl->val);
                break;
        case V4L2_CID_SATURATION:
                tpg_s_saturation(&dev->tpg, ctrl->val);
                break;
        case V4L2_CID_HUE:
                tpg_s_hue(&dev->tpg, ctrl->val);
                break;
        case V4L2_CID_HFLIP:
                dev->hflip = ctrl->val;
                tpg_s_hflip(&dev->tpg, dev->sensor_hflip ^ dev->hflip);
                break;
        case V4L2_CID_VFLIP:
                dev->vflip = ctrl->val;
                tpg_s_vflip(&dev->tpg, dev->sensor_vflip ^ dev->vflip);
                break;
        case V4L2_CID_ALPHA_COMPONENT:
                tpg_s_alpha_component(&dev->tpg, ctrl->val);
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_user_vid_ctrl_ops = {
        .g_volatile_ctrl = vivid_user_vid_g_volatile_ctrl,
        .s_ctrl = vivid_user_vid_s_ctrl,
};


/* Video Capture Controls */

static void vivid_update_power_present(struct vivid_dev *dev)
{
        unsigned int i, j;

        dev->power_present = 0;
        for (i = 0, j = 0;
             i < ARRAY_SIZE(dev->dv_timings_signal_mode); i++) {
                if (dev->input_type[i] != HDMI)
                        continue;
                /*
                 * If connected to TPG or HDMI output, and the signal
                 * mode is not NO_SIGNAL, then there is power present.
                 */
                if (dev->input_is_connected_to_output[i] != 1 &&
                    dev->dv_timings_signal_mode[i] != NO_SIGNAL)
                        dev->power_present |= (1 << j);
                j++;
        }

        __v4l2_ctrl_s_ctrl(dev->ctrl_rx_power_present,
                           dev->power_present);

        v4l2_ctrl_activate(dev->ctrl_dv_timings,
                           dev->dv_timings_signal_mode[dev->input] ==
                           SELECTED_DV_TIMINGS);
}

static int vivid_vid_cap_s_ctrl(struct v4l2_ctrl *ctrl)
{
        static const u32 colorspaces[] = {
                V4L2_COLORSPACE_SMPTE170M,
                V4L2_COLORSPACE_REC709,
                V4L2_COLORSPACE_SRGB,
                V4L2_COLORSPACE_OPRGB,
                V4L2_COLORSPACE_BT2020,
                V4L2_COLORSPACE_DCI_P3,
                V4L2_COLORSPACE_SMPTE240M,
                V4L2_COLORSPACE_470_SYSTEM_M,
                V4L2_COLORSPACE_470_SYSTEM_BG,
        };
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev, ctrl_hdl_vid_cap);
        unsigned int i;
        struct vivid_dev *output_inst = NULL;
        int index = 0;
        int hdmi_index, svid_index;
        s32 input_index = 0;

        switch (ctrl->id) {
        case VIVID_CID_TEST_PATTERN:
                vivid_update_quality(dev);
                tpg_s_pattern(&dev->tpg, ctrl->val);
                break;
        case VIVID_CID_COLORSPACE:
                tpg_s_colorspace(&dev->tpg, colorspaces[ctrl->val]);
                vivid_send_source_change(dev, TV);
                vivid_send_source_change(dev, SVID);
                vivid_send_source_change(dev, HDMI);
                vivid_send_source_change(dev, WEBCAM);
                break;
        case VIVID_CID_XFER_FUNC:
                tpg_s_xfer_func(&dev->tpg, ctrl->val);
                vivid_send_source_change(dev, TV);
                vivid_send_source_change(dev, SVID);
                vivid_send_source_change(dev, HDMI);
                vivid_send_source_change(dev, WEBCAM);
                break;
        case VIVID_CID_YCBCR_ENC:
                tpg_s_ycbcr_enc(&dev->tpg, ctrl->val);
                vivid_send_source_change(dev, TV);
                vivid_send_source_change(dev, SVID);
                vivid_send_source_change(dev, HDMI);
                vivid_send_source_change(dev, WEBCAM);
                break;
        case VIVID_CID_HSV_ENC:
                tpg_s_hsv_enc(&dev->tpg, ctrl->val ? V4L2_HSV_ENC_256 :
                                                     V4L2_HSV_ENC_180);
                vivid_send_source_change(dev, TV);
                vivid_send_source_change(dev, SVID);
                vivid_send_source_change(dev, HDMI);
                vivid_send_source_change(dev, WEBCAM);
                break;
        case VIVID_CID_QUANTIZATION:
                tpg_s_quantization(&dev->tpg, ctrl->val);
                vivid_send_source_change(dev, TV);
                vivid_send_source_change(dev, SVID);
                vivid_send_source_change(dev, HDMI);
                vivid_send_source_change(dev, WEBCAM);
                break;
        case V4L2_CID_DV_RX_RGB_RANGE:
                if (!vivid_is_hdmi_cap(dev))
                        break;
                tpg_s_rgb_range(&dev->tpg, ctrl->val);
                break;
        case VIVID_CID_LIMITED_RGB_RANGE:
                tpg_s_real_rgb_range(&dev->tpg, ctrl->val ?
                                V4L2_DV_RGB_RANGE_LIMITED : V4L2_DV_RGB_RANGE_FULL);
                break;
        case VIVID_CID_ALPHA_MODE:
                tpg_s_alpha_mode(&dev->tpg, ctrl->val);
                break;
        case VIVID_CID_HOR_MOVEMENT:
                tpg_s_mv_hor_mode(&dev->tpg, ctrl->val);
                break;
        case VIVID_CID_VERT_MOVEMENT:
                tpg_s_mv_vert_mode(&dev->tpg, ctrl->val);
                break;
        case VIVID_CID_OSD_TEXT_MODE:
                dev->osd_mode = ctrl->val;
                break;
        case VIVID_CID_PERCENTAGE_FILL:
                tpg_s_perc_fill(&dev->tpg, ctrl->val);
                for (i = 0; i < VIDEO_MAX_FRAME; i++)
                        dev->must_blank[i] = ctrl->val < 100;
                break;
        case VIVID_CID_INSERT_SAV:
                tpg_s_insert_sav(&dev->tpg, ctrl->val);
                break;
        case VIVID_CID_INSERT_EAV:
                tpg_s_insert_eav(&dev->tpg, ctrl->val);
                break;
        case VIVID_CID_INSERT_HDMI_VIDEO_GUARD_BAND:
                tpg_s_insert_hdmi_video_guard_band(&dev->tpg, ctrl->val);
                break;
        case VIVID_CID_HFLIP:
                dev->sensor_hflip = ctrl->val;
                tpg_s_hflip(&dev->tpg, dev->sensor_hflip ^ dev->hflip);
                break;
        case VIVID_CID_VFLIP:
                dev->sensor_vflip = ctrl->val;
                tpg_s_vflip(&dev->tpg, dev->sensor_vflip ^ dev->vflip);
                break;
        case VIVID_CID_REDUCED_FPS:
                dev->reduced_fps = ctrl->val;
                vivid_update_format_cap(dev, true);
                break;
        case VIVID_CID_HAS_CROP_CAP:
                dev->has_crop_cap = ctrl->val;
                vivid_update_format_cap(dev, true);
                break;
        case VIVID_CID_HAS_COMPOSE_CAP:
                dev->has_compose_cap = ctrl->val;
                vivid_update_format_cap(dev, true);
                break;
        case VIVID_CID_HAS_SCALER_CAP:
                dev->has_scaler_cap = ctrl->val;
                vivid_update_format_cap(dev, true);
                break;
        case VIVID_CID_SHOW_BORDER:
                tpg_s_show_border(&dev->tpg, ctrl->val);
                break;
        case VIVID_CID_SHOW_SQUARE:
                tpg_s_show_square(&dev->tpg, ctrl->val);
                break;
        case VIVID_CID_STD_ASPECT_RATIO:
                dev->std_aspect_ratio[dev->input] = ctrl->val;
                tpg_s_video_aspect(&dev->tpg, vivid_get_video_aspect(dev));
                break;
        case VIVID_CID_DV_TIMINGS_SIGNAL_MODE:
                dev->dv_timings_signal_mode[dev->input] =
                        dev->ctrl_dv_timings_signal_mode->val;
                dev->query_dv_timings[dev->input] = dev->ctrl_dv_timings->val;
                vivid_update_power_present(dev);
                vivid_update_quality(dev);
                vivid_send_input_source_change(dev, dev->input);
                break;
        case VIVID_CID_DV_TIMINGS_ASPECT_RATIO:
                dev->dv_timings_aspect_ratio[dev->input] = ctrl->val;
                tpg_s_video_aspect(&dev->tpg, vivid_get_video_aspect(dev));
                break;
        case VIVID_CID_TSTAMP_SRC:
                dev->tstamp_src_is_soe = ctrl->val;
                dev->vb_vid_cap_q.timestamp_flags &= ~V4L2_BUF_FLAG_TSTAMP_SRC_MASK;
                if (dev->tstamp_src_is_soe)
                        dev->vb_vid_cap_q.timestamp_flags |= V4L2_BUF_FLAG_TSTAMP_SRC_SOE;
                break;
        case VIVID_CID_MAX_EDID_BLOCKS:
                dev->edid_max_blocks = ctrl->val;
                if (dev->edid_blocks > dev->edid_max_blocks)
                        dev->edid_blocks = dev->edid_max_blocks;
                break;
        case VIVID_CID_HDMI_IS_CONNECTED_TO_OUTPUT(0) ... VIVID_CID_HDMI_IS_CONNECTED_TO_OUTPUT(14):
                hdmi_index = ctrl->id - VIVID_CID_HDMI_IS_CONNECTED_TO_OUTPUT(0);
                output_inst = vivid_ctrl_hdmi_to_output_instance[ctrl->cur.val];
                index = vivid_ctrl_hdmi_to_output_index[ctrl->cur.val];
                input_index = dev->hdmi_index_to_input_index[hdmi_index];
                dev->input_is_connected_to_output[input_index] = ctrl->val;

                if (output_inst) {
                        output_inst->output_to_input_instance[index] = NULL;
                        vivid_update_outputs(output_inst);
                        cec_phys_addr_invalidate(output_inst->cec_tx_adap[index]);
                }
                if (ctrl->val >= FIXED_MENU_ITEMS) {
                        output_inst = vivid_ctrl_hdmi_to_output_instance[ctrl->val];
                        index = vivid_ctrl_hdmi_to_output_index[ctrl->val];
                        output_inst->output_to_input_instance[index] = dev;
                        output_inst->output_to_input_index[index] =
                                dev->hdmi_index_to_input_index[hdmi_index];
                }
                spin_lock(&hdmi_output_skip_mask_lock);
                hdmi_to_output_menu_skip_mask &= ~(1ULL << ctrl->cur.val);
                if (ctrl->val >= FIXED_MENU_ITEMS)
                        hdmi_to_output_menu_skip_mask |= 1ULL << ctrl->val;
                spin_unlock(&hdmi_output_skip_mask_lock);
                vivid_update_power_present(dev);
                vivid_update_quality(dev);
                vivid_send_input_source_change(dev, dev->hdmi_index_to_input_index[hdmi_index]);
                if (ctrl->val < FIXED_MENU_ITEMS && ctrl->cur.val < FIXED_MENU_ITEMS)
                        break;
                spin_lock(&hdmi_output_skip_mask_lock);
                hdmi_input_update_outputs_mask |= 1 << dev->inst;
                spin_unlock(&hdmi_output_skip_mask_lock);
                queue_work(update_hdmi_ctrls_workqueue, &dev->update_hdmi_ctrl_work);
                break;
        case VIVID_CID_SVID_IS_CONNECTED_TO_OUTPUT(0) ... VIVID_CID_SVID_IS_CONNECTED_TO_OUTPUT(15):
                svid_index = ctrl->id - VIVID_CID_SVID_IS_CONNECTED_TO_OUTPUT(0);
                output_inst = vivid_ctrl_svid_to_output_instance[ctrl->cur.val];
                index = vivid_ctrl_svid_to_output_index[ctrl->cur.val];
                input_index = dev->svid_index_to_input_index[svid_index];
                dev->input_is_connected_to_output[input_index] = ctrl->val;

                if (output_inst)
                        output_inst->output_to_input_instance[index] = NULL;
                if (ctrl->val >= FIXED_MENU_ITEMS) {
                        output_inst = vivid_ctrl_svid_to_output_instance[ctrl->val];
                        index = vivid_ctrl_svid_to_output_index[ctrl->val];
                        output_inst->output_to_input_instance[index] = dev;
                        output_inst->output_to_input_index[index] =
                                dev->svid_index_to_input_index[svid_index];
                }
                spin_lock(&svid_output_skip_mask_lock);
                svid_to_output_menu_skip_mask &= ~(1ULL << ctrl->cur.val);
                if (ctrl->val >= FIXED_MENU_ITEMS)
                        svid_to_output_menu_skip_mask |= 1ULL << ctrl->val;
                spin_unlock(&svid_output_skip_mask_lock);
                vivid_update_quality(dev);
                vivid_send_input_source_change(dev, dev->svid_index_to_input_index[svid_index]);
                if (ctrl->val < FIXED_MENU_ITEMS && ctrl->cur.val < FIXED_MENU_ITEMS)
                        break;
                queue_work(update_svid_ctrls_workqueue, &dev->update_svid_ctrl_work);
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_vid_cap_ctrl_ops = {
        .s_ctrl = vivid_vid_cap_s_ctrl,
};

static const char * const vivid_ctrl_hor_movement_strings[] = {
        "Move Left Fast",
        "Move Left",
        "Move Left Slow",
        "No Movement",
        "Move Right Slow",
        "Move Right",
        "Move Right Fast",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_hor_movement = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_HOR_MOVEMENT,
        .name = "Horizontal Movement",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = TPG_MOVE_POS_FAST,
        .def = TPG_MOVE_NONE,
        .qmenu = vivid_ctrl_hor_movement_strings,
};

static const char * const vivid_ctrl_vert_movement_strings[] = {
        "Move Up Fast",
        "Move Up",
        "Move Up Slow",
        "No Movement",
        "Move Down Slow",
        "Move Down",
        "Move Down Fast",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_vert_movement = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_VERT_MOVEMENT,
        .name = "Vertical Movement",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = TPG_MOVE_POS_FAST,
        .def = TPG_MOVE_NONE,
        .qmenu = vivid_ctrl_vert_movement_strings,
};

static const struct v4l2_ctrl_config vivid_ctrl_show_border = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_SHOW_BORDER,
        .name = "Show Border",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_show_square = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_SHOW_SQUARE,
        .name = "Show Square",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};

static const char * const vivid_ctrl_osd_mode_strings[] = {
        "All",
        "Counters Only",
        "None",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_osd_mode = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_OSD_TEXT_MODE,
        .name = "OSD Text Mode",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = ARRAY_SIZE(vivid_ctrl_osd_mode_strings) - 2,
        .qmenu = vivid_ctrl_osd_mode_strings,
};

static const struct v4l2_ctrl_config vivid_ctrl_perc_fill = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_PERCENTAGE_FILL,
        .name = "Fill Percentage of Frame",
        .type = V4L2_CTRL_TYPE_INTEGER,
        .min = 0,
        .max = 100,
        .def = 100,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_insert_sav = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_INSERT_SAV,
        .name = "Insert SAV Code in Image",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_insert_eav = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_INSERT_EAV,
        .name = "Insert EAV Code in Image",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_insert_hdmi_video_guard_band = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_INSERT_HDMI_VIDEO_GUARD_BAND,
        .name = "Insert Video Guard Band",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_hflip = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_HFLIP,
        .name = "Sensor Flipped Horizontally",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_vflip = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_VFLIP,
        .name = "Sensor Flipped Vertically",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_reduced_fps = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_REDUCED_FPS,
        .name = "Reduced Framerate",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_has_crop_cap = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_HAS_CROP_CAP,
        .name = "Enable Capture Cropping",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .def = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_has_compose_cap = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_HAS_COMPOSE_CAP,
        .name = "Enable Capture Composing",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .def = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_has_scaler_cap = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_HAS_SCALER_CAP,
        .name = "Enable Capture Scaler",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .def = 1,
        .step = 1,
};

static const char * const vivid_ctrl_tstamp_src_strings[] = {
        "End of Frame",
        "Start of Exposure",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_tstamp_src = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_TSTAMP_SRC,
        .name = "Timestamp Source",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = ARRAY_SIZE(vivid_ctrl_tstamp_src_strings) - 2,
        .qmenu = vivid_ctrl_tstamp_src_strings,
};

static const struct v4l2_ctrl_config vivid_ctrl_std_aspect_ratio = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_STD_ASPECT_RATIO,
        .name = "Standard Aspect Ratio",
        .type = V4L2_CTRL_TYPE_MENU,
        .min = 1,
        .max = 4,
        .def = 1,
        .qmenu = tpg_aspect_strings,
};

static const char * const vivid_ctrl_dv_timings_signal_mode_strings[] = {
        "Current DV Timings",
        "No Signal",
        "No Lock",
        "Out of Range",
        "Selected DV Timings",
        "Cycle Through All DV Timings",
        "Custom DV Timings",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_dv_timings_signal_mode = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_DV_TIMINGS_SIGNAL_MODE,
        .name = "DV Timings Signal Mode",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = 5,
        .qmenu = vivid_ctrl_dv_timings_signal_mode_strings,
};

static const struct v4l2_ctrl_config vivid_ctrl_dv_timings_aspect_ratio = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_DV_TIMINGS_ASPECT_RATIO,
        .name = "DV Timings Aspect Ratio",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = 3,
        .qmenu = tpg_aspect_strings,
};

static const struct v4l2_ctrl_config vivid_ctrl_max_edid_blocks = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_MAX_EDID_BLOCKS,
        .name = "Maximum EDID Blocks",
        .type = V4L2_CTRL_TYPE_INTEGER,
        .min = 1,
        .max = 256,
        .def = 2,
        .step = 1,
};

static const char * const vivid_ctrl_colorspace_strings[] = {
        "SMPTE 170M",
        "Rec. 709",
        "sRGB",
        "opRGB",
        "BT.2020",
        "DCI-P3",
        "SMPTE 240M",
        "470 System M",
        "470 System BG",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_colorspace = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_COLORSPACE,
        .name = "Colorspace",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = ARRAY_SIZE(vivid_ctrl_colorspace_strings) - 2,
        .def = 2,
        .qmenu = vivid_ctrl_colorspace_strings,
};

static const char * const vivid_ctrl_xfer_func_strings[] = {
        "Default",
        "Rec. 709",
        "sRGB",
        "opRGB",
        "SMPTE 240M",
        "None",
        "DCI-P3",
        "SMPTE 2084",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_xfer_func = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_XFER_FUNC,
        .name = "Transfer Function",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = ARRAY_SIZE(vivid_ctrl_xfer_func_strings) - 2,
        .qmenu = vivid_ctrl_xfer_func_strings,
};

static const char * const vivid_ctrl_ycbcr_enc_strings[] = {
        "Default",
        "ITU-R 601",
        "Rec. 709",
        "xvYCC 601",
        "xvYCC 709",
        "",
        "BT.2020",
        "BT.2020 Constant Luminance",
        "SMPTE 240M",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_ycbcr_enc = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_YCBCR_ENC,
        .name = "Y'CbCr Encoding",
        .type = V4L2_CTRL_TYPE_MENU,
        .menu_skip_mask = 1 << 5,
        .max = ARRAY_SIZE(vivid_ctrl_ycbcr_enc_strings) - 2,
        .qmenu = vivid_ctrl_ycbcr_enc_strings,
};

static const char * const vivid_ctrl_hsv_enc_strings[] = {
        "Hue 0-179",
        "Hue 0-256",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_hsv_enc = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_HSV_ENC,
        .name = "HSV Encoding",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = ARRAY_SIZE(vivid_ctrl_hsv_enc_strings) - 2,
        .qmenu = vivid_ctrl_hsv_enc_strings,
};

static const char * const vivid_ctrl_quantization_strings[] = {
        "Default",
        "Full Range",
        "Limited Range",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_quantization = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_QUANTIZATION,
        .name = "Quantization",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = ARRAY_SIZE(vivid_ctrl_quantization_strings) - 2,
        .qmenu = vivid_ctrl_quantization_strings,
};

static const struct v4l2_ctrl_config vivid_ctrl_alpha_mode = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_ALPHA_MODE,
        .name = "Apply Alpha To Red Only",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_limited_rgb_range = {
        .ops = &vivid_vid_cap_ctrl_ops,
        .id = VIVID_CID_LIMITED_RGB_RANGE,
        .name = "Limited RGB Range (16-235)",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};


/* VBI Capture Control */

static int vivid_vbi_cap_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev, ctrl_hdl_vbi_cap);

        switch (ctrl->id) {
        case VIVID_CID_VBI_CAP_INTERLACED:
                dev->vbi_cap_interlaced = ctrl->val;
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_vbi_cap_ctrl_ops = {
        .s_ctrl = vivid_vbi_cap_s_ctrl,
};

static const struct v4l2_ctrl_config vivid_ctrl_vbi_cap_interlaced = {
        .ops = &vivid_vbi_cap_ctrl_ops,
        .id = VIVID_CID_VBI_CAP_INTERLACED,
        .name = "Interlaced VBI Format",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};


/* Video Output Controls */

static int vivid_vid_out_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev, ctrl_hdl_vid_out);
        struct v4l2_bt_timings *bt = &dev->dv_timings_out.bt;

        switch (ctrl->id) {
        case VIVID_CID_HAS_CROP_OUT:
                dev->has_crop_out = ctrl->val;
                vivid_update_format_out(dev);
                break;
        case VIVID_CID_HAS_COMPOSE_OUT:
                dev->has_compose_out = ctrl->val;
                vivid_update_format_out(dev);
                break;
        case VIVID_CID_HAS_SCALER_OUT:
                dev->has_scaler_out = ctrl->val;
                vivid_update_format_out(dev);
                break;
        case V4L2_CID_DV_TX_MODE:
                dev->dvi_d_out = ctrl->val == V4L2_DV_TX_MODE_DVI_D;
                if (!vivid_is_hdmi_out(dev))
                        break;
                if (!dev->dvi_d_out && (bt->flags & V4L2_DV_FL_IS_CE_VIDEO)) {
                        if (bt->width == 720 && bt->height <= 576)
                                dev->colorspace_out = V4L2_COLORSPACE_SMPTE170M;
                        else
                                dev->colorspace_out = V4L2_COLORSPACE_REC709;
                        dev->quantization_out = V4L2_QUANTIZATION_DEFAULT;
                } else {
                        dev->colorspace_out = V4L2_COLORSPACE_SRGB;
                        dev->quantization_out = dev->dvi_d_out ?
                                        V4L2_QUANTIZATION_LIM_RANGE :
                                        V4L2_QUANTIZATION_DEFAULT;
                }
                if (vivid_output_is_connected_to(dev)) {
                        struct vivid_dev *dev_rx = vivid_output_is_connected_to(dev);

                        vivid_send_source_change(dev_rx, HDMI);
                }
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_vid_out_ctrl_ops = {
        .s_ctrl = vivid_vid_out_s_ctrl,
};

static const struct v4l2_ctrl_config vivid_ctrl_has_crop_out = {
        .ops = &vivid_vid_out_ctrl_ops,
        .id = VIVID_CID_HAS_CROP_OUT,
        .name = "Enable Output Cropping",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .def = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_has_compose_out = {
        .ops = &vivid_vid_out_ctrl_ops,
        .id = VIVID_CID_HAS_COMPOSE_OUT,
        .name = "Enable Output Composing",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .def = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_has_scaler_out = {
        .ops = &vivid_vid_out_ctrl_ops,
        .id = VIVID_CID_HAS_SCALER_OUT,
        .name = "Enable Output Scaler",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .def = 1,
        .step = 1,
};

/* Streaming Controls */

static int vivid_streaming_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev, ctrl_hdl_streaming);

        switch (ctrl->id) {
        case VIVID_CID_DQBUF_ERROR:
                dev->dqbuf_error = true;
                break;
        case VIVID_CID_PERC_DROPPED:
                dev->perc_dropped_buffers = ctrl->val;
                break;
        case VIVID_CID_QUEUE_SETUP_ERROR:
                dev->queue_setup_error = true;
                break;
        case VIVID_CID_BUF_PREPARE_ERROR:
                dev->buf_prepare_error = true;
                break;
        case VIVID_CID_START_STR_ERROR:
                dev->start_streaming_error = true;
                break;
        case VIVID_CID_REQ_VALIDATE_ERROR:
                dev->req_validate_error = true;
                break;
        case VIVID_CID_QUEUE_ERROR:
                if (vb2_start_streaming_called(&dev->vb_vid_cap_q))
                        vb2_queue_error(&dev->vb_vid_cap_q);
                if (vb2_start_streaming_called(&dev->vb_vbi_cap_q))
                        vb2_queue_error(&dev->vb_vbi_cap_q);
                if (vb2_start_streaming_called(&dev->vb_vid_out_q))
                        vb2_queue_error(&dev->vb_vid_out_q);
                if (vb2_start_streaming_called(&dev->vb_vbi_out_q))
                        vb2_queue_error(&dev->vb_vbi_out_q);
                if (vb2_start_streaming_called(&dev->vb_sdr_cap_q))
                        vb2_queue_error(&dev->vb_sdr_cap_q);
                break;
        case VIVID_CID_SEQ_WRAP:
                dev->seq_wrap = ctrl->val;
                break;
        case VIVID_CID_TIME_WRAP:
                dev->time_wrap = ctrl->val;
                if (dev->time_wrap == 1)
                        dev->time_wrap = (1ULL << 63) - NSEC_PER_SEC * 16ULL;
                else if (dev->time_wrap == 2)
                        dev->time_wrap = ((1ULL << 31) - 16) * NSEC_PER_SEC;
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_streaming_ctrl_ops = {
        .s_ctrl = vivid_streaming_s_ctrl,
};

static const struct v4l2_ctrl_config vivid_ctrl_dqbuf_error = {
        .ops = &vivid_streaming_ctrl_ops,
        .id = VIVID_CID_DQBUF_ERROR,
        .name = "Inject V4L2_BUF_FLAG_ERROR",
        .type = V4L2_CTRL_TYPE_BUTTON,
};

static const struct v4l2_ctrl_config vivid_ctrl_perc_dropped = {
        .ops = &vivid_streaming_ctrl_ops,
        .id = VIVID_CID_PERC_DROPPED,
        .name = "Percentage of Dropped Buffers",
        .type = V4L2_CTRL_TYPE_INTEGER,
        .min = 0,
        .max = 100,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_queue_setup_error = {
        .ops = &vivid_streaming_ctrl_ops,
        .id = VIVID_CID_QUEUE_SETUP_ERROR,
        .name = "Inject VIDIOC_REQBUFS Error",
        .type = V4L2_CTRL_TYPE_BUTTON,
};

static const struct v4l2_ctrl_config vivid_ctrl_buf_prepare_error = {
        .ops = &vivid_streaming_ctrl_ops,
        .id = VIVID_CID_BUF_PREPARE_ERROR,
        .name = "Inject VIDIOC_QBUF Error",
        .type = V4L2_CTRL_TYPE_BUTTON,
};

static const struct v4l2_ctrl_config vivid_ctrl_start_streaming_error = {
        .ops = &vivid_streaming_ctrl_ops,
        .id = VIVID_CID_START_STR_ERROR,
        .name = "Inject VIDIOC_STREAMON Error",
        .type = V4L2_CTRL_TYPE_BUTTON,
};

static const struct v4l2_ctrl_config vivid_ctrl_queue_error = {
        .ops = &vivid_streaming_ctrl_ops,
        .id = VIVID_CID_QUEUE_ERROR,
        .name = "Inject Fatal Streaming Error",
        .type = V4L2_CTRL_TYPE_BUTTON,
};

#ifdef CONFIG_MEDIA_CONTROLLER
static const struct v4l2_ctrl_config vivid_ctrl_req_validate_error = {
        .ops = &vivid_streaming_ctrl_ops,
        .id = VIVID_CID_REQ_VALIDATE_ERROR,
        .name = "Inject req_validate() Error",
        .type = V4L2_CTRL_TYPE_BUTTON,
};
#endif

static const struct v4l2_ctrl_config vivid_ctrl_seq_wrap = {
        .ops = &vivid_streaming_ctrl_ops,
        .id = VIVID_CID_SEQ_WRAP,
        .name = "Wrap Sequence Number",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};

static const char * const vivid_ctrl_time_wrap_strings[] = {
        "None",
        "64 Bit",
        "32 Bit",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_time_wrap = {
        .ops = &vivid_streaming_ctrl_ops,
        .id = VIVID_CID_TIME_WRAP,
        .name = "Wrap Timestamp",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = ARRAY_SIZE(vivid_ctrl_time_wrap_strings) - 2,
        .qmenu = vivid_ctrl_time_wrap_strings,
};


/* SDTV Capture Controls */

static int vivid_sdtv_cap_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev, ctrl_hdl_sdtv_cap);

        switch (ctrl->id) {
        case VIVID_CID_STD_SIGNAL_MODE:
                dev->std_signal_mode[dev->input] =
                        dev->ctrl_std_signal_mode->val;
                if (dev->std_signal_mode[dev->input] == SELECTED_STD)
                        dev->query_std[dev->input] =
                                vivid_standard[dev->ctrl_standard->val];
                v4l2_ctrl_activate(dev->ctrl_standard,
                                   dev->std_signal_mode[dev->input] ==
                                        SELECTED_STD);
                vivid_update_quality(dev);
                vivid_send_source_change(dev, TV);
                vivid_send_source_change(dev, SVID);
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_sdtv_cap_ctrl_ops = {
        .s_ctrl = vivid_sdtv_cap_s_ctrl,
};

static const char * const vivid_ctrl_std_signal_mode_strings[] = {
        "Current Standard",
        "No Signal",
        "No Lock",
        "",
        "Selected Standard",
        "Cycle Through All Standards",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_std_signal_mode = {
        .ops = &vivid_sdtv_cap_ctrl_ops,
        .id = VIVID_CID_STD_SIGNAL_MODE,
        .name = "Standard Signal Mode",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = ARRAY_SIZE(vivid_ctrl_std_signal_mode_strings) - 2,
        .menu_skip_mask = 1 << 3,
        .qmenu = vivid_ctrl_std_signal_mode_strings,
};

static const struct v4l2_ctrl_config vivid_ctrl_standard = {
        .ops = &vivid_sdtv_cap_ctrl_ops,
        .id = VIVID_CID_STANDARD,
        .name = "Standard",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = 14,
        .qmenu = vivid_ctrl_standard_strings,
};



/* Radio Receiver Controls */

static int vivid_radio_rx_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev, ctrl_hdl_radio_rx);

        switch (ctrl->id) {
        case VIVID_CID_RADIO_SEEK_MODE:
                dev->radio_rx_hw_seek_mode = ctrl->val;
                break;
        case VIVID_CID_RADIO_SEEK_PROG_LIM:
                dev->radio_rx_hw_seek_prog_lim = ctrl->val;
                break;
        case VIVID_CID_RADIO_RX_RDS_RBDS:
                dev->rds_gen.use_rbds = ctrl->val;
                break;
        case VIVID_CID_RADIO_RX_RDS_BLOCKIO:
                dev->radio_rx_rds_controls = ctrl->val;
                dev->radio_rx_caps &= ~V4L2_CAP_READWRITE;
                dev->radio_rx_rds_use_alternates = false;
                if (!dev->radio_rx_rds_controls) {
                        dev->radio_rx_caps |= V4L2_CAP_READWRITE;
                        __v4l2_ctrl_s_ctrl(dev->radio_rx_rds_pty, 0);
                        __v4l2_ctrl_s_ctrl(dev->radio_rx_rds_ta, 0);
                        __v4l2_ctrl_s_ctrl(dev->radio_rx_rds_tp, 0);
                        __v4l2_ctrl_s_ctrl(dev->radio_rx_rds_ms, 0);
                        __v4l2_ctrl_s_ctrl_string(dev->radio_rx_rds_psname, "");
                        __v4l2_ctrl_s_ctrl_string(dev->radio_rx_rds_radiotext, "");
                }
                v4l2_ctrl_activate(dev->radio_rx_rds_pty, dev->radio_rx_rds_controls);
                v4l2_ctrl_activate(dev->radio_rx_rds_psname, dev->radio_rx_rds_controls);
                v4l2_ctrl_activate(dev->radio_rx_rds_radiotext, dev->radio_rx_rds_controls);
                v4l2_ctrl_activate(dev->radio_rx_rds_ta, dev->radio_rx_rds_controls);
                v4l2_ctrl_activate(dev->radio_rx_rds_tp, dev->radio_rx_rds_controls);
                v4l2_ctrl_activate(dev->radio_rx_rds_ms, dev->radio_rx_rds_controls);
                dev->radio_rx_dev.device_caps = dev->radio_rx_caps;
                break;
        case V4L2_CID_RDS_RECEPTION:
                dev->radio_rx_rds_enabled = ctrl->val;
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_radio_rx_ctrl_ops = {
        .s_ctrl = vivid_radio_rx_s_ctrl,
};

static const char * const vivid_ctrl_radio_rds_mode_strings[] = {
        "Block I/O",
        "Controls",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_radio_rx_rds_blockio = {
        .ops = &vivid_radio_rx_ctrl_ops,
        .id = VIVID_CID_RADIO_RX_RDS_BLOCKIO,
        .name = "RDS Rx I/O Mode",
        .type = V4L2_CTRL_TYPE_MENU,
        .qmenu = vivid_ctrl_radio_rds_mode_strings,
        .max = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_radio_rx_rds_rbds = {
        .ops = &vivid_radio_rx_ctrl_ops,
        .id = VIVID_CID_RADIO_RX_RDS_RBDS,
        .name = "Generate RBDS Instead of RDS",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};

static const char * const vivid_ctrl_radio_hw_seek_mode_strings[] = {
        "Bounded",
        "Wrap Around",
        "Both",
        NULL,
};

static const struct v4l2_ctrl_config vivid_ctrl_radio_hw_seek_mode = {
        .ops = &vivid_radio_rx_ctrl_ops,
        .id = VIVID_CID_RADIO_SEEK_MODE,
        .name = "Radio HW Seek Mode",
        .type = V4L2_CTRL_TYPE_MENU,
        .max = 2,
        .qmenu = vivid_ctrl_radio_hw_seek_mode_strings,
};

static const struct v4l2_ctrl_config vivid_ctrl_radio_hw_seek_prog_lim = {
        .ops = &vivid_radio_rx_ctrl_ops,
        .id = VIVID_CID_RADIO_SEEK_PROG_LIM,
        .name = "Radio Programmable HW Seek",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .step = 1,
};


/* Radio Transmitter Controls */

static int vivid_radio_tx_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev, ctrl_hdl_radio_tx);

        switch (ctrl->id) {
        case VIVID_CID_RADIO_TX_RDS_BLOCKIO:
                dev->radio_tx_rds_controls = ctrl->val;
                dev->radio_tx_caps &= ~V4L2_CAP_READWRITE;
                if (!dev->radio_tx_rds_controls)
                        dev->radio_tx_caps |= V4L2_CAP_READWRITE;
                dev->radio_tx_dev.device_caps = dev->radio_tx_caps;
                break;
        case V4L2_CID_RDS_TX_PTY:
                if (dev->radio_rx_rds_controls)
                        v4l2_ctrl_s_ctrl(dev->radio_rx_rds_pty, ctrl->val);
                break;
        case V4L2_CID_RDS_TX_PS_NAME:
                if (dev->radio_rx_rds_controls)
                        v4l2_ctrl_s_ctrl_string(dev->radio_rx_rds_psname, ctrl->p_new.p_char);
                break;
        case V4L2_CID_RDS_TX_RADIO_TEXT:
                if (dev->radio_rx_rds_controls)
                        v4l2_ctrl_s_ctrl_string(dev->radio_rx_rds_radiotext, ctrl->p_new.p_char);
                break;
        case V4L2_CID_RDS_TX_TRAFFIC_ANNOUNCEMENT:
                if (dev->radio_rx_rds_controls)
                        v4l2_ctrl_s_ctrl(dev->radio_rx_rds_ta, ctrl->val);
                break;
        case V4L2_CID_RDS_TX_TRAFFIC_PROGRAM:
                if (dev->radio_rx_rds_controls)
                        v4l2_ctrl_s_ctrl(dev->radio_rx_rds_tp, ctrl->val);
                break;
        case V4L2_CID_RDS_TX_MUSIC_SPEECH:
                if (dev->radio_rx_rds_controls)
                        v4l2_ctrl_s_ctrl(dev->radio_rx_rds_ms, ctrl->val);
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_radio_tx_ctrl_ops = {
        .s_ctrl = vivid_radio_tx_s_ctrl,
};

static const struct v4l2_ctrl_config vivid_ctrl_radio_tx_rds_blockio = {
        .ops = &vivid_radio_tx_ctrl_ops,
        .id = VIVID_CID_RADIO_TX_RDS_BLOCKIO,
        .name = "RDS Tx I/O Mode",
        .type = V4L2_CTRL_TYPE_MENU,
        .qmenu = vivid_ctrl_radio_rds_mode_strings,
        .max = 1,
        .def = 1,
};


/* SDR Capture Controls */

static int vivid_sdr_cap_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev, ctrl_hdl_sdr_cap);

        switch (ctrl->id) {
        case VIVID_CID_SDR_CAP_FM_DEVIATION:
                dev->sdr_fm_deviation = ctrl->val;
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_sdr_cap_ctrl_ops = {
        .s_ctrl = vivid_sdr_cap_s_ctrl,
};

static const struct v4l2_ctrl_config vivid_ctrl_sdr_cap_fm_deviation = {
        .ops = &vivid_sdr_cap_ctrl_ops,
        .id = VIVID_CID_SDR_CAP_FM_DEVIATION,
        .name = "FM Deviation",
        .type = V4L2_CTRL_TYPE_INTEGER,
        .min =    100,
        .max = 200000,
        .def =  75000,
        .step =     1,
};

/* Metadata Capture Control */

static int vivid_meta_cap_s_ctrl(struct v4l2_ctrl *ctrl)
{
        struct vivid_dev *dev = container_of(ctrl->handler, struct vivid_dev,
                                             ctrl_hdl_meta_cap);

        switch (ctrl->id) {
        case VIVID_CID_META_CAP_GENERATE_PTS:
                dev->meta_pts = ctrl->val;
                break;
        case VIVID_CID_META_CAP_GENERATE_SCR:
                dev->meta_scr = ctrl->val;
                break;
        }
        return 0;
}

static const struct v4l2_ctrl_ops vivid_meta_cap_ctrl_ops = {
        .s_ctrl = vivid_meta_cap_s_ctrl,
};

static const struct v4l2_ctrl_config vivid_ctrl_meta_has_pts = {
        .ops = &vivid_meta_cap_ctrl_ops,
        .id = VIVID_CID_META_CAP_GENERATE_PTS,
        .name = "Generate PTS",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .def = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_meta_has_src_clk = {
        .ops = &vivid_meta_cap_ctrl_ops,
        .id = VIVID_CID_META_CAP_GENERATE_SCR,
        .name = "Generate SCR",
        .type = V4L2_CTRL_TYPE_BOOLEAN,
        .max = 1,
        .def = 1,
        .step = 1,
};

static const struct v4l2_ctrl_config vivid_ctrl_class = {
        .ops = &vivid_user_gen_ctrl_ops,
        .flags = V4L2_CTRL_FLAG_READ_ONLY | V4L2_CTRL_FLAG_WRITE_ONLY,
        .id = VIVID_CID_VIVID_CLASS,
        .name = "Vivid Controls",
        .type = V4L2_CTRL_TYPE_CTRL_CLASS,
};

int vivid_create_controls(struct vivid_dev *dev, bool show_ccs_cap,
                bool show_ccs_out, bool no_error_inj,
                bool has_sdtv, bool has_hdmi)
{
        struct v4l2_ctrl_handler *hdl_user_gen = &dev->ctrl_hdl_user_gen;
        struct v4l2_ctrl_handler *hdl_user_vid = &dev->ctrl_hdl_user_vid;
        struct v4l2_ctrl_handler *hdl_user_aud = &dev->ctrl_hdl_user_aud;
        struct v4l2_ctrl_handler *hdl_streaming = &dev->ctrl_hdl_streaming;
        struct v4l2_ctrl_handler *hdl_sdtv_cap = &dev->ctrl_hdl_sdtv_cap;
        struct v4l2_ctrl_handler *hdl_loop_cap = &dev->ctrl_hdl_loop_cap;
        struct v4l2_ctrl_handler *hdl_fb = &dev->ctrl_hdl_fb;
        struct v4l2_ctrl_handler *hdl_vid_cap = &dev->ctrl_hdl_vid_cap;
        struct v4l2_ctrl_handler *hdl_vid_out = &dev->ctrl_hdl_vid_out;
        struct v4l2_ctrl_handler *hdl_vbi_cap = &dev->ctrl_hdl_vbi_cap;
        struct v4l2_ctrl_handler *hdl_vbi_out = &dev->ctrl_hdl_vbi_out;
        struct v4l2_ctrl_handler *hdl_radio_rx = &dev->ctrl_hdl_radio_rx;
        struct v4l2_ctrl_handler *hdl_radio_tx = &dev->ctrl_hdl_radio_tx;
        struct v4l2_ctrl_handler *hdl_sdr_cap = &dev->ctrl_hdl_sdr_cap;
        struct v4l2_ctrl_handler *hdl_meta_cap = &dev->ctrl_hdl_meta_cap;
        struct v4l2_ctrl_handler *hdl_meta_out = &dev->ctrl_hdl_meta_out;
        struct v4l2_ctrl_handler *hdl_tch_cap = &dev->ctrl_hdl_touch_cap;

        struct v4l2_ctrl_config vivid_ctrl_dv_timings = {
                .ops = &vivid_vid_cap_ctrl_ops,
                .id = VIVID_CID_DV_TIMINGS,
                .name = "DV Timings",
                .type = V4L2_CTRL_TYPE_MENU,
        };
        int i;

        v4l2_ctrl_handler_init(hdl_user_gen, 10);
        v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_user_vid, 9);
        v4l2_ctrl_new_custom(hdl_user_vid, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_user_aud, 2);
        v4l2_ctrl_new_custom(hdl_user_aud, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_streaming, 8);
        v4l2_ctrl_new_custom(hdl_streaming, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_sdtv_cap, 2);
        v4l2_ctrl_new_custom(hdl_sdtv_cap, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_loop_cap, 1);
        v4l2_ctrl_new_custom(hdl_loop_cap, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_fb, 1);
        v4l2_ctrl_new_custom(hdl_fb, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_vid_cap, 55);
        v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_vid_out, 26);
        if (!no_error_inj || dev->has_fb || dev->num_hdmi_outputs)
                v4l2_ctrl_new_custom(hdl_vid_out, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_vbi_cap, 21);
        v4l2_ctrl_new_custom(hdl_vbi_cap, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_vbi_out, 19);
        if (!no_error_inj)
                v4l2_ctrl_new_custom(hdl_vbi_out, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_radio_rx, 17);
        v4l2_ctrl_new_custom(hdl_radio_rx, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_radio_tx, 17);
        v4l2_ctrl_new_custom(hdl_radio_tx, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_sdr_cap, 19);
        v4l2_ctrl_new_custom(hdl_sdr_cap, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_meta_cap, 2);
        v4l2_ctrl_new_custom(hdl_meta_cap, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_meta_out, 2);
        v4l2_ctrl_new_custom(hdl_meta_out, &vivid_ctrl_class, NULL);
        v4l2_ctrl_handler_init(hdl_tch_cap, 2);
        v4l2_ctrl_new_custom(hdl_tch_cap, &vivid_ctrl_class, NULL);

        /* User Controls */
        dev->volume = v4l2_ctrl_new_std(hdl_user_aud, NULL,
                V4L2_CID_AUDIO_VOLUME, 0, 255, 1, 200);
        dev->mute = v4l2_ctrl_new_std(hdl_user_aud, NULL,
                V4L2_CID_AUDIO_MUTE, 0, 1, 1, 0);
        if (dev->has_vid_cap) {
                dev->brightness = v4l2_ctrl_new_std(hdl_user_vid, &vivid_user_vid_ctrl_ops,
                        V4L2_CID_BRIGHTNESS, 0, 255, 1, 128);
                for (i = 0; i < MAX_INPUTS; i++)
                        dev->input_brightness[i] = 128;
                dev->contrast = v4l2_ctrl_new_std(hdl_user_vid, &vivid_user_vid_ctrl_ops,
                        V4L2_CID_CONTRAST, 0, 255, 1, 128);
                dev->saturation = v4l2_ctrl_new_std(hdl_user_vid, &vivid_user_vid_ctrl_ops,
                        V4L2_CID_SATURATION, 0, 255, 1, 128);
                dev->hue = v4l2_ctrl_new_std(hdl_user_vid, &vivid_user_vid_ctrl_ops,
                        V4L2_CID_HUE, -128, 128, 1, 0);
                v4l2_ctrl_new_std(hdl_user_vid, &vivid_user_vid_ctrl_ops,
                        V4L2_CID_HFLIP, 0, 1, 1, 0);
                v4l2_ctrl_new_std(hdl_user_vid, &vivid_user_vid_ctrl_ops,
                        V4L2_CID_VFLIP, 0, 1, 1, 0);
                dev->autogain = v4l2_ctrl_new_std(hdl_user_vid, &vivid_user_vid_ctrl_ops,
                        V4L2_CID_AUTOGAIN, 0, 1, 1, 1);
                dev->gain = v4l2_ctrl_new_std(hdl_user_vid, &vivid_user_vid_ctrl_ops,
                        V4L2_CID_GAIN, 0, 255, 1, 100);
                dev->alpha = v4l2_ctrl_new_std(hdl_user_vid, &vivid_user_vid_ctrl_ops,
                        V4L2_CID_ALPHA_COMPONENT, 0, 255, 1, 0);
        }
        dev->button = v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_button, NULL);
        dev->int32 = v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_int32, NULL);
        dev->int64 = v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_int64, NULL);
        dev->boolean = v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_boolean, NULL);
        dev->menu = v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_menu, NULL);
        dev->string = v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_string, NULL);
        dev->bitmask = v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_bitmask, NULL);
        dev->int_menu = v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_int_menu, NULL);
        dev->ro_int32 = v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_ro_int32, NULL);
        v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_area, NULL);
        v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_u32_array, NULL);
        v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_u32_dyn_array, NULL);
        v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_u16_matrix, NULL);
        v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_u8_4d_array, NULL);
        dev->pixel_array = v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_u8_pixel_array, NULL);
        v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_s32_array, NULL);
        v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_s64_array, NULL);

        if (dev->has_vid_cap) {
                /* Image Processing Controls */
                struct v4l2_ctrl_config vivid_ctrl_test_pattern = {
                        .ops = &vivid_vid_cap_ctrl_ops,
                        .id = VIVID_CID_TEST_PATTERN,
                        .name = "Test Pattern",
                        .type = V4L2_CTRL_TYPE_MENU,
                        .max = TPG_PAT_NOISE,
                        .qmenu = tpg_pattern_strings,
                };

                dev->test_pattern = v4l2_ctrl_new_custom(hdl_vid_cap,
                                &vivid_ctrl_test_pattern, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_perc_fill, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_hor_movement, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_vert_movement, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_osd_mode, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_show_border, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_show_square, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_hflip, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_vflip, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_insert_sav, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_insert_eav, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_insert_hdmi_video_guard_band, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_reduced_fps, NULL);

                WARN_ON(dev->num_hdmi_inputs > MAX_HDMI_INPUTS);
                WARN_ON(dev->num_svid_inputs > MAX_SVID_INPUTS);

                for (u8 i = 0; i < dev->num_hdmi_inputs; i++) {
                        snprintf(dev->ctrl_hdmi_to_output_names[i],
                                 sizeof(dev->ctrl_hdmi_to_output_names[i]),
                                 "HDMI %03u-%u Is Connected To", dev->inst, i);
                }

                for (u8 i = 0; i < dev->num_hdmi_inputs; i++) {
                        struct v4l2_ctrl_config ctrl_config = {
                                .ops = &vivid_vid_cap_ctrl_ops,
                                .id = VIVID_CID_HDMI_IS_CONNECTED_TO_OUTPUT(i),
                                .name = dev->ctrl_hdmi_to_output_names[i],
                                .type = V4L2_CTRL_TYPE_MENU,
                                .max = 1,
                                .qmenu = (const char * const *)vivid_ctrl_hdmi_to_output_strings,
                        };
                        dev->ctrl_hdmi_to_output[i] = v4l2_ctrl_new_custom(hdl_vid_cap,
                                                                           &ctrl_config, NULL);
                }

                for (u8 i = 0; i < dev->num_svid_inputs; i++) {
                        snprintf(dev->ctrl_svid_to_output_names[i],
                                 sizeof(dev->ctrl_svid_to_output_names[i]),
                                 "S-Video %03u-%u Is Connected To", dev->inst, i);
                }

                for (u8 i = 0; i < dev->num_svid_inputs; i++) {
                        struct v4l2_ctrl_config ctrl_config = {
                                .ops = &vivid_vid_cap_ctrl_ops,
                                .id = VIVID_CID_SVID_IS_CONNECTED_TO_OUTPUT(i),
                                .name = dev->ctrl_svid_to_output_names[i],
                                .type = V4L2_CTRL_TYPE_MENU,
                                .max = 1,
                                .qmenu = (const char * const *)vivid_ctrl_svid_to_output_strings,
                        };
                        dev->ctrl_svid_to_output[i] = v4l2_ctrl_new_custom(hdl_vid_cap,
                                                                           &ctrl_config, NULL);
                }

                if (show_ccs_cap) {
                        dev->ctrl_has_crop_cap = v4l2_ctrl_new_custom(hdl_vid_cap,
                                &vivid_ctrl_has_crop_cap, NULL);
                        dev->ctrl_has_compose_cap = v4l2_ctrl_new_custom(hdl_vid_cap,
                                &vivid_ctrl_has_compose_cap, NULL);
                        dev->ctrl_has_scaler_cap = v4l2_ctrl_new_custom(hdl_vid_cap,
                                &vivid_ctrl_has_scaler_cap, NULL);
                }

                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_tstamp_src, NULL);
                dev->colorspace = v4l2_ctrl_new_custom(hdl_vid_cap,
                        &vivid_ctrl_colorspace, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_xfer_func, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_ycbcr_enc, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_hsv_enc, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_quantization, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_alpha_mode, NULL);
        }

        if (dev->has_vid_out && show_ccs_out) {
                dev->ctrl_has_crop_out = v4l2_ctrl_new_custom(hdl_vid_out,
                        &vivid_ctrl_has_crop_out, NULL);
                dev->ctrl_has_compose_out = v4l2_ctrl_new_custom(hdl_vid_out,
                        &vivid_ctrl_has_compose_out, NULL);
                dev->ctrl_has_scaler_out = v4l2_ctrl_new_custom(hdl_vid_out,
                        &vivid_ctrl_has_scaler_out, NULL);
        }

        /*
         * Testing this driver with v4l2-compliance will trigger the error
         * injection controls, and after that nothing will work as expected.
         * So we have a module option to drop these error injecting controls
         * allowing us to run v4l2_compliance again.
         */
        if (!no_error_inj) {
                v4l2_ctrl_new_custom(hdl_user_gen, &vivid_ctrl_disconnect, NULL);
                v4l2_ctrl_new_custom(hdl_streaming, &vivid_ctrl_dqbuf_error, NULL);
                v4l2_ctrl_new_custom(hdl_streaming, &vivid_ctrl_perc_dropped, NULL);
                v4l2_ctrl_new_custom(hdl_streaming, &vivid_ctrl_queue_setup_error, NULL);
                v4l2_ctrl_new_custom(hdl_streaming, &vivid_ctrl_buf_prepare_error, NULL);
                v4l2_ctrl_new_custom(hdl_streaming, &vivid_ctrl_start_streaming_error, NULL);
                v4l2_ctrl_new_custom(hdl_streaming, &vivid_ctrl_queue_error, NULL);
#ifdef CONFIG_MEDIA_CONTROLLER
                v4l2_ctrl_new_custom(hdl_streaming, &vivid_ctrl_req_validate_error, NULL);
#endif
                v4l2_ctrl_new_custom(hdl_streaming, &vivid_ctrl_seq_wrap, NULL);
                v4l2_ctrl_new_custom(hdl_streaming, &vivid_ctrl_time_wrap, NULL);
        }

        if (has_sdtv && (dev->has_vid_cap || dev->has_vbi_cap)) {
                if (dev->has_vid_cap)
                        v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_std_aspect_ratio, NULL);
                dev->ctrl_std_signal_mode = v4l2_ctrl_new_custom(hdl_sdtv_cap,
                        &vivid_ctrl_std_signal_mode, NULL);
                dev->ctrl_standard = v4l2_ctrl_new_custom(hdl_sdtv_cap,
                        &vivid_ctrl_standard, NULL);
                if (dev->ctrl_std_signal_mode)
                        v4l2_ctrl_cluster(2, &dev->ctrl_std_signal_mode);
                if (dev->has_raw_vbi_cap)
                        v4l2_ctrl_new_custom(hdl_vbi_cap, &vivid_ctrl_vbi_cap_interlaced, NULL);
        }

        if (dev->num_hdmi_inputs) {
                s64 hdmi_input_mask = GENMASK(dev->num_hdmi_inputs - 1, 0);

                dev->ctrl_dv_timings_signal_mode = v4l2_ctrl_new_custom(hdl_vid_cap,
                                        &vivid_ctrl_dv_timings_signal_mode, NULL);

                vivid_ctrl_dv_timings.max = dev->query_dv_timings_size - 1;
                vivid_ctrl_dv_timings.qmenu =
                        (const char * const *)dev->query_dv_timings_qmenu;
                dev->ctrl_dv_timings = v4l2_ctrl_new_custom(hdl_vid_cap,
                        &vivid_ctrl_dv_timings, NULL);
                if (dev->ctrl_dv_timings_signal_mode)
                        v4l2_ctrl_cluster(2, &dev->ctrl_dv_timings_signal_mode);

                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_dv_timings_aspect_ratio, NULL);
                v4l2_ctrl_new_custom(hdl_vid_cap, &vivid_ctrl_max_edid_blocks, NULL);
                dev->real_rgb_range_cap = v4l2_ctrl_new_custom(hdl_vid_cap,
                        &vivid_ctrl_limited_rgb_range, NULL);
                dev->rgb_range_cap = v4l2_ctrl_new_std_menu(hdl_vid_cap,
                        &vivid_vid_cap_ctrl_ops,
                        V4L2_CID_DV_RX_RGB_RANGE, V4L2_DV_RGB_RANGE_FULL,
                        0, V4L2_DV_RGB_RANGE_AUTO);
                dev->ctrl_rx_power_present = v4l2_ctrl_new_std(hdl_vid_cap,
                        NULL, V4L2_CID_DV_RX_POWER_PRESENT, 0, hdmi_input_mask,
                        0, hdmi_input_mask);

        }
        if (dev->num_hdmi_outputs) {
                s64 hdmi_output_mask = GENMASK(dev->num_hdmi_outputs - 1, 0);

                /*
                 * We aren't doing anything with this at the moment, but
                 * HDMI outputs typically have this controls.
                 */
                dev->ctrl_tx_rgb_range = v4l2_ctrl_new_std_menu(hdl_vid_out, NULL,
                        V4L2_CID_DV_TX_RGB_RANGE, V4L2_DV_RGB_RANGE_FULL,
                        0, V4L2_DV_RGB_RANGE_AUTO);
                dev->ctrl_tx_mode = v4l2_ctrl_new_std_menu(hdl_vid_out, NULL,
                        V4L2_CID_DV_TX_MODE, V4L2_DV_TX_MODE_HDMI,
                        0, V4L2_DV_TX_MODE_HDMI);
                dev->ctrl_tx_hotplug = v4l2_ctrl_new_std(hdl_vid_out, NULL,
                        V4L2_CID_DV_TX_HOTPLUG, 0, hdmi_output_mask, 0, 0);
                dev->ctrl_tx_rxsense = v4l2_ctrl_new_std(hdl_vid_out, NULL,
                        V4L2_CID_DV_TX_RXSENSE, 0, hdmi_output_mask, 0, 0);
                dev->ctrl_tx_edid_present = v4l2_ctrl_new_std(hdl_vid_out, NULL,
                        V4L2_CID_DV_TX_EDID_PRESENT, 0, hdmi_output_mask, 0, 0);
        }

        if (dev->has_fb)
                v4l2_ctrl_new_custom(hdl_fb, &vivid_ctrl_clear_fb, NULL);

        if (dev->has_radio_rx) {
                v4l2_ctrl_new_custom(hdl_radio_rx, &vivid_ctrl_radio_hw_seek_mode, NULL);
                v4l2_ctrl_new_custom(hdl_radio_rx, &vivid_ctrl_radio_hw_seek_prog_lim, NULL);
                v4l2_ctrl_new_custom(hdl_radio_rx, &vivid_ctrl_radio_rx_rds_blockio, NULL);
                v4l2_ctrl_new_custom(hdl_radio_rx, &vivid_ctrl_radio_rx_rds_rbds, NULL);
                v4l2_ctrl_new_std(hdl_radio_rx, &vivid_radio_rx_ctrl_ops,
                        V4L2_CID_RDS_RECEPTION, 0, 1, 1, 1);
                dev->radio_rx_rds_pty = v4l2_ctrl_new_std(hdl_radio_rx,
                        &vivid_radio_rx_ctrl_ops,
                        V4L2_CID_RDS_RX_PTY, 0, 31, 1, 0);
                dev->radio_rx_rds_psname = v4l2_ctrl_new_std(hdl_radio_rx,
                        &vivid_radio_rx_ctrl_ops,
                        V4L2_CID_RDS_RX_PS_NAME, 0, 8, 8, 0);
                dev->radio_rx_rds_radiotext = v4l2_ctrl_new_std(hdl_radio_rx,
                        &vivid_radio_rx_ctrl_ops,
                        V4L2_CID_RDS_RX_RADIO_TEXT, 0, 64, 64, 0);
                dev->radio_rx_rds_ta = v4l2_ctrl_new_std(hdl_radio_rx,
                        &vivid_radio_rx_ctrl_ops,
                        V4L2_CID_RDS_RX_TRAFFIC_ANNOUNCEMENT, 0, 1, 1, 0);
                dev->radio_rx_rds_tp = v4l2_ctrl_new_std(hdl_radio_rx,
                        &vivid_radio_rx_ctrl_ops,
                        V4L2_CID_RDS_RX_TRAFFIC_PROGRAM, 0, 1, 1, 0);
                dev->radio_rx_rds_ms = v4l2_ctrl_new_std(hdl_radio_rx,
                        &vivid_radio_rx_ctrl_ops,
                        V4L2_CID_RDS_RX_MUSIC_SPEECH, 0, 1, 1, 1);
        }
        if (dev->has_radio_tx) {
                v4l2_ctrl_new_custom(hdl_radio_tx,
                        &vivid_ctrl_radio_tx_rds_blockio, NULL);
                dev->radio_tx_rds_pi = v4l2_ctrl_new_std(hdl_radio_tx,
                        &vivid_radio_tx_ctrl_ops,
                        V4L2_CID_RDS_TX_PI, 0, 0xffff, 1, 0x8088);
                dev->radio_tx_rds_pty = v4l2_ctrl_new_std(hdl_radio_tx,
                        &vivid_radio_tx_ctrl_ops,
                        V4L2_CID_RDS_TX_PTY, 0, 31, 1, 3);
                dev->radio_tx_rds_psname = v4l2_ctrl_new_std(hdl_radio_tx,
                        &vivid_radio_tx_ctrl_ops,
                        V4L2_CID_RDS_TX_PS_NAME, 0, 8, 8, 0);
                if (dev->radio_tx_rds_psname)
                        v4l2_ctrl_s_ctrl_string(dev->radio_tx_rds_psname, "VIVID-TX");
                dev->radio_tx_rds_radiotext = v4l2_ctrl_new_std(hdl_radio_tx,
                        &vivid_radio_tx_ctrl_ops,
                        V4L2_CID_RDS_TX_RADIO_TEXT, 0, 64 * 2, 64, 0);
                if (dev->radio_tx_rds_radiotext)
                        v4l2_ctrl_s_ctrl_string(dev->radio_tx_rds_radiotext,
                               "This is a VIVID default Radio Text template text, change at will");
                dev->radio_tx_rds_mono_stereo = v4l2_ctrl_new_std(hdl_radio_tx,
                        &vivid_radio_tx_ctrl_ops,
                        V4L2_CID_RDS_TX_MONO_STEREO, 0, 1, 1, 1);
                dev->radio_tx_rds_art_head = v4l2_ctrl_new_std(hdl_radio_tx,
                        &vivid_radio_tx_ctrl_ops,
                        V4L2_CID_RDS_TX_ARTIFICIAL_HEAD, 0, 1, 1, 0);
                dev->radio_tx_rds_compressed = v4l2_ctrl_new_std(hdl_radio_tx,
                        &vivid_radio_tx_ctrl_ops,
                        V4L2_CID_RDS_TX_COMPRESSED, 0, 1, 1, 0);
                dev->radio_tx_rds_dyn_pty = v4l2_ctrl_new_std(hdl_radio_tx,
                        &vivid_radio_tx_ctrl_ops,
                        V4L2_CID_RDS_TX_DYNAMIC_PTY, 0, 1, 1, 0);
                dev->radio_tx_rds_ta = v4l2_ctrl_new_std(hdl_radio_tx,
                        &vivid_radio_tx_ctrl_ops,
                        V4L2_CID_RDS_TX_TRAFFIC_ANNOUNCEMENT, 0, 1, 1, 0);
                dev->radio_tx_rds_tp = v4l2_ctrl_new_std(hdl_radio_tx,
                        &vivid_radio_tx_ctrl_ops,
                        V4L2_CID_RDS_TX_TRAFFIC_PROGRAM, 0, 1, 1, 1);
                dev->radio_tx_rds_ms = v4l2_ctrl_new_std(hdl_radio_tx,
                        &vivid_radio_tx_ctrl_ops,
                        V4L2_CID_RDS_TX_MUSIC_SPEECH, 0, 1, 1, 1);
        }
        if (dev->has_sdr_cap) {
                v4l2_ctrl_new_custom(hdl_sdr_cap,
                        &vivid_ctrl_sdr_cap_fm_deviation, NULL);
        }
        if (dev->has_meta_cap) {
                v4l2_ctrl_new_custom(hdl_meta_cap,
                                     &vivid_ctrl_meta_has_pts, NULL);
                v4l2_ctrl_new_custom(hdl_meta_cap,
                                     &vivid_ctrl_meta_has_src_clk, NULL);
        }

        if (hdl_user_gen->error)
                return hdl_user_gen->error;
        if (hdl_user_vid->error)
                return hdl_user_vid->error;
        if (hdl_user_aud->error)
                return hdl_user_aud->error;
        if (hdl_streaming->error)
                return hdl_streaming->error;
        if (hdl_sdr_cap->error)
                return hdl_sdr_cap->error;
        if (hdl_loop_cap->error)
                return hdl_loop_cap->error;

        if (dev->autogain)
                v4l2_ctrl_auto_cluster(2, &dev->autogain, 0, true);

        if (dev->has_vid_cap) {
                v4l2_ctrl_add_handler(hdl_vid_cap, hdl_user_gen, NULL, false);
                v4l2_ctrl_add_handler(hdl_vid_cap, hdl_user_vid, NULL, false);
                v4l2_ctrl_add_handler(hdl_vid_cap, hdl_user_aud, NULL, false);
                v4l2_ctrl_add_handler(hdl_vid_cap, hdl_streaming, NULL, false);
                v4l2_ctrl_add_handler(hdl_vid_cap, hdl_sdtv_cap, NULL, false);
                v4l2_ctrl_add_handler(hdl_vid_cap, hdl_loop_cap, NULL, false);
                v4l2_ctrl_add_handler(hdl_vid_cap, hdl_fb, NULL, false);
                if (hdl_vid_cap->error)
                        return hdl_vid_cap->error;
                dev->vid_cap_dev.ctrl_handler = hdl_vid_cap;
        }
        if (dev->has_vid_out) {
                v4l2_ctrl_add_handler(hdl_vid_out, hdl_user_gen, NULL, false);
                v4l2_ctrl_add_handler(hdl_vid_out, hdl_user_aud, NULL, false);
                v4l2_ctrl_add_handler(hdl_vid_out, hdl_streaming, NULL, false);
                v4l2_ctrl_add_handler(hdl_vid_out, hdl_fb, NULL, false);
                if (hdl_vid_out->error)
                        return hdl_vid_out->error;
                dev->vid_out_dev.ctrl_handler = hdl_vid_out;
        }
        if (dev->has_vbi_cap) {
                v4l2_ctrl_add_handler(hdl_vbi_cap, hdl_user_gen, NULL, false);
                v4l2_ctrl_add_handler(hdl_vbi_cap, hdl_streaming, NULL, false);
                v4l2_ctrl_add_handler(hdl_vbi_cap, hdl_sdtv_cap, NULL, false);
                v4l2_ctrl_add_handler(hdl_vbi_cap, hdl_loop_cap, NULL, false);
                if (hdl_vbi_cap->error)
                        return hdl_vbi_cap->error;
                dev->vbi_cap_dev.ctrl_handler = hdl_vbi_cap;
        }
        if (dev->has_vbi_out) {
                v4l2_ctrl_add_handler(hdl_vbi_out, hdl_user_gen, NULL, false);
                v4l2_ctrl_add_handler(hdl_vbi_out, hdl_streaming, NULL, false);
                if (hdl_vbi_out->error)
                        return hdl_vbi_out->error;
                dev->vbi_out_dev.ctrl_handler = hdl_vbi_out;
        }
        if (dev->has_radio_rx) {
                v4l2_ctrl_add_handler(hdl_radio_rx, hdl_user_gen, NULL, false);
                v4l2_ctrl_add_handler(hdl_radio_rx, hdl_user_aud, NULL, false);
                if (hdl_radio_rx->error)
                        return hdl_radio_rx->error;
                dev->radio_rx_dev.ctrl_handler = hdl_radio_rx;
        }
        if (dev->has_radio_tx) {
                v4l2_ctrl_add_handler(hdl_radio_tx, hdl_user_gen, NULL, false);
                v4l2_ctrl_add_handler(hdl_radio_tx, hdl_user_aud, NULL, false);
                if (hdl_radio_tx->error)
                        return hdl_radio_tx->error;
                dev->radio_tx_dev.ctrl_handler = hdl_radio_tx;
        }
        if (dev->has_sdr_cap) {
                v4l2_ctrl_add_handler(hdl_sdr_cap, hdl_user_gen, NULL, false);
                v4l2_ctrl_add_handler(hdl_sdr_cap, hdl_streaming, NULL, false);
                if (hdl_sdr_cap->error)
                        return hdl_sdr_cap->error;
                dev->sdr_cap_dev.ctrl_handler = hdl_sdr_cap;
        }
        if (dev->has_meta_cap) {
                v4l2_ctrl_add_handler(hdl_meta_cap, hdl_user_gen, NULL, false);
                v4l2_ctrl_add_handler(hdl_meta_cap, hdl_streaming, NULL, false);
                if (hdl_meta_cap->error)
                        return hdl_meta_cap->error;
                dev->meta_cap_dev.ctrl_handler = hdl_meta_cap;
        }
        if (dev->has_meta_out) {
                v4l2_ctrl_add_handler(hdl_meta_out, hdl_user_gen, NULL, false);
                v4l2_ctrl_add_handler(hdl_meta_out, hdl_streaming, NULL, false);
                if (hdl_meta_out->error)
                        return hdl_meta_out->error;
                dev->meta_out_dev.ctrl_handler = hdl_meta_out;
        }
        if (dev->has_touch_cap) {
                v4l2_ctrl_add_handler(hdl_tch_cap, hdl_user_gen, NULL, false);
                v4l2_ctrl_add_handler(hdl_tch_cap, hdl_streaming, NULL, false);
                if (hdl_tch_cap->error)
                        return hdl_tch_cap->error;
                dev->touch_cap_dev.ctrl_handler = hdl_tch_cap;
        }
        return 0;
}

void vivid_free_controls(struct vivid_dev *dev)
{
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_vid_cap);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_vid_out);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_vbi_cap);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_vbi_out);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_radio_rx);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_radio_tx);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_sdr_cap);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_user_gen);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_user_vid);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_user_aud);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_streaming);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_sdtv_cap);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_loop_cap);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_fb);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_meta_cap);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_meta_out);
        v4l2_ctrl_handler_free(&dev->ctrl_hdl_touch_cap);
}







































   12 







   12 




















   12 

   11 







   12 






   10 










































































































































































































































































































































































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
// SPDX-License-Identifier: GPL-2.0
/******************************************************************************
 * rtl871x_recv.c
 *
 * Copyright(c) 2007 - 2010 Realtek Corporation. All rights reserved.
 * Linux device driver for RTL8192SU
 *
 * Modifications for inclusion into the Linux staging tree are
 * Copyright(c) 2010 Larry Finger. All rights reserved.
 *
 * Contact information:
 * WLAN FAE <wlanfae@realtek.com>
 * Larry Finger <Larry.Finger@lwfinger.net>
 *
 ******************************************************************************/

#define _RTL871X_RECV_C_

#include <linux/ip.h>
#include <linux/if_ether.h>
#include <linux/etherdevice.h>
#include <linux/ieee80211.h>
#include <net/cfg80211.h>

#include "osdep_service.h"
#include "drv_types.h"
#include "recv_osdep.h"
#include "mlme_osdep.h"
#include "ethernet.h"
#include "usb_ops.h"
#include "wifi.h"

static const u8 SNAP_ETH_TYPE_IPX[2] = {0x81, 0x37};

/* Datagram Delivery Protocol */
static const u8 SNAP_ETH_TYPE_APPLETALK_AARP[2] = {0x80, 0xf3};

void _r8712_init_sta_recv_priv(struct sta_recv_priv *psta_recvpriv)
{
        memset((u8 *)psta_recvpriv, 0, sizeof(struct sta_recv_priv));
        spin_lock_init(&psta_recvpriv->lock);
        _init_queue(&psta_recvpriv->defrag_q);
}

int _r8712_init_recv_priv(struct recv_priv *precvpriv,
                          struct _adapter *padapter)
{
        int ret;
        sint i;
        union recv_frame *precvframe;

        memset((unsigned char *)precvpriv, 0, sizeof(struct  recv_priv));
        spin_lock_init(&precvpriv->lock);
        _init_queue(&precvpriv->free_recv_queue);
        _init_queue(&precvpriv->recv_pending_queue);
        precvpriv->adapter = padapter;
        precvpriv->free_recvframe_cnt = NR_RECVFRAME;
        precvpriv->pallocated_frame_buf = kzalloc(NR_RECVFRAME *
                                sizeof(union recv_frame) + RXFRAME_ALIGN_SZ,
                                GFP_ATOMIC);
        if (!precvpriv->pallocated_frame_buf)
                return -ENOMEM;
        precvpriv->precv_frame_buf = precvpriv->pallocated_frame_buf +
                                    RXFRAME_ALIGN_SZ -
                                    ((addr_t)(precvpriv->pallocated_frame_buf) &
                                    (RXFRAME_ALIGN_SZ - 1));
        precvframe = (union recv_frame *)precvpriv->precv_frame_buf;
        for (i = 0; i < NR_RECVFRAME; i++) {
                INIT_LIST_HEAD(&(precvframe->u.list));
                list_add_tail(&(precvframe->u.list),
                              &(precvpriv->free_recv_queue.queue));
                r8712_os_recv_resource_alloc(padapter, precvframe);
                precvframe->u.hdr.adapter = padapter;
                precvframe++;
        }
        precvpriv->rx_pending_cnt = 1;
        ret = r8712_init_recv_priv(precvpriv, padapter);
        if (ret)
                kfree(precvpriv->pallocated_frame_buf);

        return ret;
}

void _r8712_free_recv_priv(struct recv_priv *precvpriv)
{
        kfree(precvpriv->pallocated_frame_buf);
        r8712_free_recv_priv(precvpriv);
}

union recv_frame *r8712_alloc_recvframe(struct __queue *pfree_recv_queue)
{
        unsigned long irqL;
        union recv_frame  *precvframe;
        struct _adapter *padapter;
        struct recv_priv *precvpriv;

        spin_lock_irqsave(&pfree_recv_queue->lock, irqL);
        precvframe = list_first_entry_or_null(&pfree_recv_queue->queue,
                                              union recv_frame, u.hdr.list);
        if (precvframe) {
                list_del_init(&precvframe->u.hdr.list);
                padapter = precvframe->u.hdr.adapter;
                if (padapter) {
                        precvpriv = &padapter->recvpriv;
                        if (pfree_recv_queue == &precvpriv->free_recv_queue)
                                precvpriv->free_recvframe_cnt--;
                }
        }
        spin_unlock_irqrestore(&pfree_recv_queue->lock, irqL);
        return precvframe;
}

/*
 * caller : defrag; recvframe_chk_defrag in recv_thread  (passive)
 * pframequeue: defrag_queue : will be accessed in recv_thread  (passive)
 * using spin_lock to protect
 */
void r8712_free_recvframe_queue(struct  __queue *pframequeue,
                                struct  __queue *pfree_recv_queue)
{
        union        recv_frame *precvframe;
        struct list_head *plist, *phead;

        spin_lock(&pframequeue->lock);
        phead = &pframequeue->queue;
        plist = phead->next;
        while (!end_of_queue_search(phead, plist)) {
                precvframe = container_of(plist, union recv_frame, u.list);
                plist = plist->next;
                r8712_free_recvframe(precvframe, pfree_recv_queue);
        }
        spin_unlock(&pframequeue->lock);
}

sint r8712_recvframe_chkmic(struct _adapter *adapter,
                            union recv_frame *precvframe)
{
        sint i, res = _SUCCESS;
        u32        datalen;
        u8 miccode[8];
        u8 bmic_err = false;
        u8 *pframe, *payload, *pframemic;
        u8   *mickey, idx, *iv;
        struct        sta_info *stainfo;
        struct        rx_pkt_attrib *prxattrib = &precvframe->u.hdr.attrib;
        struct        security_priv *psecuritypriv = &adapter->securitypriv;

        stainfo = r8712_get_stainfo(&adapter->stapriv, &prxattrib->ta[0]);
        if (prxattrib->encrypt == _TKIP_) {
                /* calculate mic code */
                if (stainfo) {
                        if (is_multicast_ether_addr(prxattrib->ra)) {
                                iv = precvframe->u.hdr.rx_data +
                                     prxattrib->hdrlen;
                                idx = iv[3];
                                mickey = &psecuritypriv->XGrprxmickey[(((idx >>
                                         6) & 0x3)) - 1].skey[0];
                                if (!psecuritypriv->binstallGrpkey)
                                        return _FAIL;
                        } else {
                                mickey = &stainfo->tkiprxmickey.skey[0];
                        }
                        /*icv_len included the mic code*/
                        datalen = precvframe->u.hdr.len - prxattrib->hdrlen -
                                  prxattrib->iv_len - prxattrib->icv_len - 8;
                        pframe = precvframe->u.hdr.rx_data;
                        payload = pframe + prxattrib->hdrlen +
                                  prxattrib->iv_len;
                        seccalctkipmic(mickey, pframe, payload, datalen,
                                       &miccode[0],
                                       (unsigned char)prxattrib->priority);
                        pframemic = payload + datalen;
                        bmic_err = false;
                        for (i = 0; i < 8; i++) {
                                if (miccode[i] != *(pframemic + i))
                                        bmic_err = true;
                        }
                        if (bmic_err) {
                                if (prxattrib->bdecrypted)
                                        r8712_handle_tkip_mic_err(adapter,
                                                                  (u8)is_multicast_ether_addr(prxattrib->ra));
                                res = _FAIL;
                        } else {
                                /* mic checked ok */
                                if (!psecuritypriv->bcheck_grpkey &&
                                    is_multicast_ether_addr(prxattrib->ra))
                                        psecuritypriv->bcheck_grpkey = true;
                        }
                        recvframe_pull_tail(precvframe, 8);
                }
        }
        return res;
}

/* decrypt and set the ivlen,icvlen of the recv_frame */
union recv_frame *r8712_decryptor(struct _adapter *padapter,
                                  union recv_frame *precv_frame)
{
        struct rx_pkt_attrib *prxattrib = &precv_frame->u.hdr.attrib;
        struct security_priv *psecuritypriv = &padapter->securitypriv;
        union recv_frame *return_packet = precv_frame;

        if ((prxattrib->encrypt > 0) && ((prxattrib->bdecrypted == 0) ||
                                         psecuritypriv->sw_decrypt)) {
                psecuritypriv->hw_decrypted = false;
                switch (prxattrib->encrypt) {
                case _WEP40_:
                case _WEP104_:
                        r8712_wep_decrypt(padapter, (u8 *)precv_frame);
                        break;
                case _TKIP_:
                        r8712_tkip_decrypt(padapter, (u8 *)precv_frame);
                        break;
                case _AES_:
                        r8712_aes_decrypt(padapter, (u8 *)precv_frame);
                        break;
                default:
                                break;
                }
        } else if (prxattrib->bdecrypted == 1) {
                psecuritypriv->hw_decrypted = true;
        }
        return return_packet;
}

/*###set the security information in the recv_frame */
union recv_frame *r8712_portctrl(struct _adapter *adapter,
                                 union recv_frame *precv_frame)
{
        u8 *psta_addr, *ptr;
        uint auth_alg;
        struct recv_frame_hdr *pfhdr;
        struct sta_info *psta;
        struct        sta_priv *pstapriv;
        union recv_frame *prtnframe;
        u16 ether_type;

        pstapriv = &adapter->stapriv;
        ptr = precv_frame->u.hdr.rx_data;
        pfhdr = &precv_frame->u.hdr;
        psta_addr = pfhdr->attrib.ta;
        psta = r8712_get_stainfo(pstapriv, psta_addr);
        auth_alg = adapter->securitypriv.AuthAlgrthm;
        if (auth_alg == 2) {
                /* get ether_type */
                ptr = ptr + pfhdr->attrib.hdrlen + LLC_HEADER_SIZE;
                ether_type = get_unaligned_be16(ptr);

                if (psta && psta->ieee8021x_blocked) {
                        /* blocked
                         * only accept EAPOL frame
                         */
                        if (ether_type == 0x888e) {
                                prtnframe = precv_frame;
                        } else {
                                /*free this frame*/
                                r8712_free_recvframe(precv_frame,
                                                     &adapter->recvpriv.free_recv_queue);
                                prtnframe = NULL;
                        }
                } else {
                        /* allowed
                         * check decryption status, and decrypt the
                         * frame if needed
                         */
                        prtnframe = precv_frame;
                        /* check is the EAPOL frame or not (Rekey) */
                        if (ether_type == 0x888e) {
                                /* check Rekey */
                                prtnframe = precv_frame;
                        }
                }
        } else {
                prtnframe = precv_frame;
        }
        return prtnframe;
}

static sint recv_decache(union recv_frame *precv_frame, u8 bretry,
                         struct stainfo_rxcache *prxcache)
{
        sint tid = precv_frame->u.hdr.attrib.priority;
        u16 seq_ctrl = ((precv_frame->u.hdr.attrib.seq_num & 0xffff) << 4) |
                        (precv_frame->u.hdr.attrib.frag_num & 0xf);

        if (tid > 15)
                return _FAIL;
        if (seq_ctrl == prxcache->tid_rxseq[tid])
                return _FAIL;
        prxcache->tid_rxseq[tid] = seq_ctrl;
        return _SUCCESS;
}

static sint sta2sta_data_frame(struct _adapter *adapter,
                               union recv_frame *precv_frame,
                               struct sta_info **psta)
{
        u8 *ptr = precv_frame->u.hdr.rx_data;
        sint ret = _SUCCESS;
        struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
        struct        sta_priv *pstapriv = &adapter->stapriv;
        struct        mlme_priv *pmlmepriv = &adapter->mlmepriv;
        u8 *mybssid  = get_bssid(pmlmepriv);
        u8 *myhwaddr = myid(&adapter->eeprompriv);
        u8 *sta_addr = NULL;
        bool bmcast = is_multicast_ether_addr(pattrib->dst);

        if (check_fwstate(pmlmepriv, WIFI_ADHOC_STATE) ||
            check_fwstate(pmlmepriv, WIFI_ADHOC_MASTER_STATE)) {
                /* filter packets that SA is myself or multicast or broadcast */
                if (!memcmp(myhwaddr, pattrib->src, ETH_ALEN))
                        return _FAIL;
                if ((memcmp(myhwaddr, pattrib->dst, ETH_ALEN)) && (!bmcast))
                        return _FAIL;
                if (is_zero_ether_addr(pattrib->bssid) ||
                    is_zero_ether_addr(mybssid) ||
                    (memcmp(pattrib->bssid, mybssid, ETH_ALEN)))
                        return _FAIL;
                sta_addr = pattrib->src;
        } else if (check_fwstate(pmlmepriv, WIFI_STATION_STATE)) {
                /* For Station mode, sa and bssid should always be BSSID,
                 * and DA is my mac-address
                 */
                if (memcmp(pattrib->bssid, pattrib->src, ETH_ALEN))
                        return _FAIL;
                sta_addr = pattrib->bssid;
        } else if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
                if (bmcast) {
                        /* For AP mode, if DA == MCAST, then BSSID should
                         * be also MCAST
                         */
                        if (!is_multicast_ether_addr(pattrib->bssid))
                                return _FAIL;
                } else { /* not mc-frame */
                        /* For AP mode, if DA is non-MCAST, then it must be
                         * BSSID, and bssid == BSSID
                         */
                        if (memcmp(pattrib->bssid, pattrib->dst, ETH_ALEN))
                                return _FAIL;
                        sta_addr = pattrib->src;
                }
        } else if (check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
                memcpy(pattrib->dst, GetAddr1Ptr(ptr), ETH_ALEN);
                memcpy(pattrib->src, GetAddr2Ptr(ptr), ETH_ALEN);
                memcpy(pattrib->bssid, GetAddr3Ptr(ptr), ETH_ALEN);
                memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);
                memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
                sta_addr = mybssid;
        } else {
                ret  = _FAIL;
        }
        if (bmcast)
                *psta = r8712_get_bcmc_stainfo(adapter);
        else
                *psta = r8712_get_stainfo(pstapriv, sta_addr); /* get ap_info */
        if (!*psta) {
                if (check_fwstate(pmlmepriv, WIFI_MP_STATE))
                        adapter->mppriv.rx_pktloss++;
                return _FAIL;
        }
        return ret;
}

static sint ap2sta_data_frame(struct _adapter *adapter,
                              union recv_frame *precv_frame,
                              struct sta_info **psta)
{
        u8 *ptr = precv_frame->u.hdr.rx_data;
        struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
        struct        sta_priv *pstapriv = &adapter->stapriv;
        struct        mlme_priv *pmlmepriv = &adapter->mlmepriv;
        u8 *mybssid  = get_bssid(pmlmepriv);
        u8 *myhwaddr = myid(&adapter->eeprompriv);
        bool bmcast = is_multicast_ether_addr(pattrib->dst);

        if (check_fwstate(pmlmepriv, WIFI_STATION_STATE) &&
            check_fwstate(pmlmepriv, _FW_LINKED)) {
                /* if NULL-frame, drop packet */
                if ((GetFrameSubType(ptr)) == (IEEE80211_FTYPE_DATA | IEEE80211_STYPE_NULLFUNC))
                        return _FAIL;
                /* drop QoS-SubType Data, including QoS NULL,
                 * excluding QoS-Data
                 */
                if ((GetFrameSubType(ptr) & WIFI_QOS_DATA_TYPE) ==
                     WIFI_QOS_DATA_TYPE) {
                        if (GetFrameSubType(ptr) & (BIT(4) | BIT(5) | BIT(6)))
                                return _FAIL;
                }

                /* filter packets that SA is myself or multicast or broadcast */
                if (!memcmp(myhwaddr, pattrib->src, ETH_ALEN))
                        return _FAIL;

                /* da should be for me */
                if ((memcmp(myhwaddr, pattrib->dst, ETH_ALEN)) && (!bmcast))
                        return _FAIL;
                /* check BSSID */
                if (is_zero_ether_addr(pattrib->bssid) ||
                    is_zero_ether_addr(mybssid) ||
                     (memcmp(pattrib->bssid, mybssid, ETH_ALEN)))
                        return _FAIL;
                if (bmcast)
                        *psta = r8712_get_bcmc_stainfo(adapter);
                else
                        *psta = r8712_get_stainfo(pstapriv, pattrib->bssid);
                if (!*psta)
                        return _FAIL;
        } else if (check_fwstate(pmlmepriv, WIFI_MP_STATE) &&
                   check_fwstate(pmlmepriv, _FW_LINKED)) {
                memcpy(pattrib->dst, GetAddr1Ptr(ptr), ETH_ALEN);
                memcpy(pattrib->src, GetAddr2Ptr(ptr), ETH_ALEN);
                memcpy(pattrib->bssid, GetAddr3Ptr(ptr), ETH_ALEN);
                memcpy(pattrib->ra, pattrib->dst, ETH_ALEN);
                memcpy(pattrib->ta, pattrib->src, ETH_ALEN);
                memcpy(pattrib->bssid,  mybssid, ETH_ALEN);
                *psta = r8712_get_stainfo(pstapriv, pattrib->bssid);
                if (!*psta)
                        return _FAIL;
        } else {
                return _FAIL;
        }
        return _SUCCESS;
}

static sint sta2ap_data_frame(struct _adapter *adapter,
                              union recv_frame *precv_frame,
                              struct sta_info **psta)
{
        struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
        struct        sta_priv *pstapriv = &adapter->stapriv;
        struct        mlme_priv *pmlmepriv = &adapter->mlmepriv;
        unsigned char *mybssid  = get_bssid(pmlmepriv);

        if (check_fwstate(pmlmepriv, WIFI_AP_STATE)) {
                /* For AP mode, if DA is non-MCAST, then it must be BSSID,
                 * and bssid == BSSID
                 * For AP mode, RA=BSSID, TX=STA(SRC_ADDR), A3=DST_ADDR
                 */
                if (memcmp(pattrib->bssid, mybssid, ETH_ALEN))
                        return _FAIL;
                *psta = r8712_get_stainfo(pstapriv, pattrib->src);
                if (!*psta)
                        return _FAIL;
        }
        return _SUCCESS;
}

static sint validate_recv_ctrl_frame(struct _adapter *adapter,
                                     union recv_frame *precv_frame)
{
        return _FAIL;
}

static sint validate_recv_mgnt_frame(struct _adapter *adapter,
                                     union recv_frame *precv_frame)
{
        return _FAIL;
}

static sint validate_recv_data_frame(struct _adapter *adapter,
                                     union recv_frame *precv_frame)
{
        int res;
        u8 bretry;
        u8 *psa, *pda, *pbssid;
        struct sta_info *psta = NULL;
        u8 *ptr = precv_frame->u.hdr.rx_data;
        struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;
        struct security_priv *psecuritypriv = &adapter->securitypriv;

        bretry = GetRetry(ptr);
        pda = ieee80211_get_DA((struct ieee80211_hdr *)ptr);
        psa = ieee80211_get_SA((struct ieee80211_hdr *)ptr);
        pbssid = get_hdr_bssid(ptr);
        if (!pbssid)
                return _FAIL;
        memcpy(pattrib->dst, pda, ETH_ALEN);
        memcpy(pattrib->src, psa, ETH_ALEN);
        memcpy(pattrib->bssid, pbssid, ETH_ALEN);
        switch (pattrib->to_fr_ds) {
        case 0:
                memcpy(pattrib->ra, pda, ETH_ALEN);
                memcpy(pattrib->ta, psa, ETH_ALEN);
                res = sta2sta_data_frame(adapter, precv_frame, &psta);
                break;
        case 1:
                memcpy(pattrib->ra, pda, ETH_ALEN);
                memcpy(pattrib->ta, pbssid, ETH_ALEN);
                res = ap2sta_data_frame(adapter, precv_frame, &psta);
                break;
        case 2:
                memcpy(pattrib->ra, pbssid, ETH_ALEN);
                memcpy(pattrib->ta, psa, ETH_ALEN);
                res = sta2ap_data_frame(adapter, precv_frame, &psta);
                break;
        case 3:
                memcpy(pattrib->ra, GetAddr1Ptr(ptr), ETH_ALEN);
                memcpy(pattrib->ta, GetAddr2Ptr(ptr), ETH_ALEN);
                return _FAIL;
        default:
                return _FAIL;
        }
        if (res == _FAIL)
                return _FAIL;
        if (!psta)
                return _FAIL;
        precv_frame->u.hdr.psta = psta;
        pattrib->amsdu = 0;
        /* parsing QC field */
        if (pattrib->qos == 1) {
                pattrib->priority = GetPriority((ptr + 24));
                pattrib->ack_policy = GetAckpolicy((ptr + 24));
                pattrib->amsdu = GetAMsdu((ptr + 24));
                pattrib->hdrlen = pattrib->to_fr_ds == 3 ? 32 : 26;
        } else {
                pattrib->priority = 0;
                pattrib->hdrlen = (pattrib->to_fr_ds == 3) ? 30 : 24;
        }

        if (pattrib->order)/*HT-CTRL 11n*/
                pattrib->hdrlen += 4;
        precv_frame->u.hdr.preorder_ctrl =
                         &psta->recvreorder_ctrl[pattrib->priority];

        /* decache, drop duplicate recv packets */
        if (recv_decache(precv_frame, bretry, &psta->sta_recvpriv.rxcache) ==
            _FAIL)
                return _FAIL;

        if (pattrib->privacy) {
                GET_ENCRY_ALGO(psecuritypriv, psta, pattrib->encrypt,
                               is_multicast_ether_addr(pattrib->ra));
                SET_ICE_IV_LEN(pattrib->iv_len, pattrib->icv_len,
                               pattrib->encrypt);
        } else {
                pattrib->encrypt = 0;
                pattrib->iv_len = pattrib->icv_len = 0;
        }
        return _SUCCESS;
}

sint r8712_validate_recv_frame(struct _adapter *adapter,
                               union recv_frame *precv_frame)
{
        /*shall check frame subtype, to / from ds, da, bssid */
        /*then call check if rx seq/frag. duplicated.*/

        u8 type;
        u8 subtype;
        sint retval = _SUCCESS;
        struct rx_pkt_attrib *pattrib = &precv_frame->u.hdr.attrib;

        u8 *ptr = precv_frame->u.hdr.rx_data;
        u8  ver = (unsigned char)(*ptr) & 0x3;

        /*add version chk*/
        if (ver != 0)
                return _FAIL;
        type =  GetFrameType(ptr);
        subtype = GetFrameSubType(ptr); /*bit(7)~bit(2)*/
        pattrib->to_fr_ds = get_tofr_ds(ptr);
        pattrib->frag_num = GetFragNum(ptr);
        pattrib->seq_num = GetSequence(ptr);
        pattrib->pw_save = GetPwrMgt(ptr);
        pattrib->mfrag = GetMFrag(ptr);
        pattrib->mdata = GetMData(ptr);
        pattrib->privacy =  GetPrivacy(ptr);
        pattrib->order = GetOrder(ptr);
        switch (type) {
        case IEEE80211_FTYPE_MGMT:
                retval = validate_recv_mgnt_frame(adapter, precv_frame);
                break;
        case IEEE80211_FTYPE_CTL:
                retval = validate_recv_ctrl_frame(adapter, precv_frame);
                break;
        case IEEE80211_FTYPE_DATA:
                pattrib->qos = (subtype & BIT(7)) ? 1 : 0;
                retval = validate_recv_data_frame(adapter, precv_frame);
                break;
        default:
                return _FAIL;
        }
        return retval;
}

int r8712_wlanhdr_to_ethhdr(union recv_frame *precvframe)
{
        /*remove the wlanhdr and add the eth_hdr*/
        sint        rmv_len;
        u16        len;
        u8        bsnaphdr;
        u8        *psnap_type;
        struct ieee80211_snap_hdr *psnap;
        struct _adapter        *adapter = precvframe->u.hdr.adapter;
        struct mlme_priv *pmlmepriv = &adapter->mlmepriv;

        u8 *ptr = precvframe->u.hdr.rx_data; /*point to frame_ctrl field*/
        struct rx_pkt_attrib *pattrib = &precvframe->u.hdr.attrib;

        if (pattrib->encrypt)
                recvframe_pull_tail(precvframe, pattrib->icv_len);
        psnap = (struct ieee80211_snap_hdr *)(ptr + pattrib->hdrlen +
                 pattrib->iv_len);
        psnap_type = ptr + pattrib->hdrlen + pattrib->iv_len + SNAP_SIZE;
        /* convert hdr + possible LLC headers into Ethernet header */
        if ((!memcmp(psnap, (void *)rfc1042_header, SNAP_SIZE) &&
             (memcmp(psnap_type, (void *)SNAP_ETH_TYPE_IPX, 2)) &&
            (memcmp(psnap_type, (void *)SNAP_ETH_TYPE_APPLETALK_AARP, 2))) ||
             !memcmp(psnap, (void *)bridge_tunnel_header, SNAP_SIZE)) {
                /* remove RFC1042 or Bridge-Tunnel encapsulation and
                 * replace EtherType
                 */
                bsnaphdr = true;
        } else {
                /* Leave Ethernet header part of hdr and full payload */
                bsnaphdr = false;
        }
        rmv_len = pattrib->hdrlen + pattrib->iv_len +
                  (bsnaphdr ? SNAP_SIZE : 0);
        len = precvframe->u.hdr.len - rmv_len;
        if (check_fwstate(pmlmepriv, WIFI_MP_STATE)) {
                ptr += rmv_len;
                *ptr = 0x87;
                *(ptr + 1) = 0x12;
                /* append rx status for mp test packets */
                ptr = recvframe_pull(precvframe, (rmv_len -
                      sizeof(struct ethhdr) + 2) - 24);
                if (!ptr)
                        return -ENOMEM;
                memcpy(ptr, get_rxmem(precvframe), 24);
                ptr += 24;
        } else {
                ptr = recvframe_pull(precvframe, (rmv_len -
                      sizeof(struct ethhdr) + (bsnaphdr ? 2 : 0)));
                if (!ptr)
                        return -ENOMEM;
        }

        memcpy(ptr, pattrib->dst, ETH_ALEN);
        memcpy(ptr + ETH_ALEN, pattrib->src, ETH_ALEN);
        if (!bsnaphdr) {
                __be16 be_tmp = htons(len);

                memcpy(ptr + 12, &be_tmp, 2);
        }
        return 0;
}

void r8712_recv_entry(union recv_frame *precvframe)
{
        struct _adapter *padapter;
        struct recv_priv *precvpriv;

        s32 ret = _SUCCESS;

        padapter = precvframe->u.hdr.adapter;
        precvpriv = &(padapter->recvpriv);

        padapter->ledpriv.LedControlHandler(padapter, LED_CTL_RX);

        ret = recv_func(padapter, precvframe);
        if (ret == _FAIL)
                goto _recv_entry_drop;
        precvpriv->rx_pkts++;
        precvpriv->rx_bytes += (uint)(precvframe->u.hdr.rx_tail -
                                precvframe->u.hdr.rx_data);
        return;
_recv_entry_drop:
        precvpriv->rx_drop++;
        padapter->mppriv.rx_pktloss = precvpriv->rx_drop;
}











































































































    3 











































   93 



    3 
   90 


   27 









   19 












   39 












   11 






















   43 




























    6 



























    6 

   19 




































































   42 






   43 





   43 






   43 









































   13 





   13 




   13 






















    2 


















    2 





































































































































































    8 



    8 
    8 


    8 








    8 
    8 






















































    8 









    8 







   42 























    4 


    4 

















   43 




   43 
    3 



   42 



  242 




    1 




  242 



















   43 





   43 

   43 


   43 


    2 




   43 


   18 
   25 




   42 














   42 

















































   29 
    8 


























































































































































































































































































































































































   22 




    5 























   21 





































































































    8 



    8 



















    8 









    8 


    2 

    6 

























    3 
    1 

   13 

    3 



    1 


   12 



    1 
    1 



    1 


    1 










   12 





    2 



   10 



   10 



    2 















    2 





    2 




































    1 


    3 




















    1 















   49 


    3 





   51 





   50 




























































































































































































    6 






































































































































































   36 
















   35 







   39 









   39 


   39 







   37 






   37 











   83 





   89 



   89 
   19 



   80 

   80 

   79 
    1 



   80 











    1 








   89 








   50 




   50 
    3 
   42 



   50 


   50 




   49 



   48 




















































































































































































































































































































































































































































































































































































































































































    1 

    1 




















































































































    9 


    9 

    9 

    9 





    8 





    8 





    8 





    8 












    8 
    8 
    7 


    5 





    8 






    8 

































































    6 








    3 
















    3 













    6 




    6 













  166 
















  265 




    6 
  259 



    1 



  256 



    6 






    6 
    6 










  256 



    6 







    6 












    6 
  253 













    1 
    6 















































































    5 


    2 







    4 





    4 
    4 

































   10 


   10 




    2 

    7 









    3 










































    3 


    3 











    3 


    3 









    6 

    6 










    6 







    3 




    3 
    1 
    2 



    1 
    1 




    2 



    3 






    2 































    6 








    6 





    6 



    3 



    3 

    3 


    3 



    3 


    3 




    3 


    1 





    3 








    1 

































    7 






    7 



    4 


    3 

















    1 


    2 































    2 




































    1 


    1 





















    1 






















   25 





























    3 


    1 
    2 



    1 








    3 
    1 














   47 








    5 
    7 
    1 
    3 

    3 




   16 



   14 
   14 


    4 



    4 



    2 


    6 


    2 



























   62 






   40 
   11 
   29 
   40 





    9 



   17 




    1 





    9 


    6 


    1 
















  120 








  118 










    3 








   11 

   47 



   63 



























































   95 











































   40 




   82 















   15 





   15 

















   15 
   15 





   30 





   30 

















   30 
   30 








    3 


    4 



    2 
   17 

   17 




   16 
   16 







    1 



    1 

    1 



   15 


















  155 






   60 





    5 

   88 













   36 
   36 






    1 







    2 





    2 





    1 


    1 
    2 




    6 




    1 



    2 







    3 






    3 

    2 




    3 






















   26 
   26 







    1 



    1 





   25 




   29 





   29 
    8 
   29 



   29 






    6 




    5 











    1 














   48 
















    2 
    1 




    8 





    8 



    1 
    1 









    1 


    1 





    1 








    3 








    3 




























    6 





    6 


    6 





    4 
















    4 






    4 


    4 







    7 





    2 









    5 
































    5 


























    9 





    3 









    6 






    6 













    4 

    2 







    2 









    5 

    2 







    3 









    4 


    1 
    3 











   10 


    5 
    5 

































































































































































































































































































  142 



  141 













   78 
   79 







   29 





   12 







   12 













   12 





























   79 







   79 








   29 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Bridge multicast support.
 *
 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
 */

#include <linux/err.h>
#include <linux/export.h>
#include <linux/if_ether.h>
#include <linux/igmp.h>
#include <linux/in.h>
#include <linux/jhash.h>
#include <linux/kernel.h>
#include <linux/log2.h>
#include <linux/netdevice.h>
#include <linux/netfilter_bridge.h>
#include <linux/random.h>
#include <linux/rculist.h>
#include <linux/skbuff.h>
#include <linux/slab.h>
#include <linux/timer.h>
#include <linux/inetdevice.h>
#include <linux/mroute.h>
#include <net/ip.h>
#include <net/switchdev.h>
#if IS_ENABLED(CONFIG_IPV6)
#include <linux/icmpv6.h>
#include <net/ipv6.h>
#include <net/mld.h>
#include <net/ip6_checksum.h>
#include <net/addrconf.h>
#endif
#include <trace/events/bridge.h>

#include "br_private.h"
#include "br_private_mcast_eht.h"

static const struct rhashtable_params br_mdb_rht_params = {
        .head_offset = offsetof(struct net_bridge_mdb_entry, rhnode),
        .key_offset = offsetof(struct net_bridge_mdb_entry, addr),
        .key_len = sizeof(struct br_ip),
        .automatic_shrinking = true,
};

static const struct rhashtable_params br_sg_port_rht_params = {
        .head_offset = offsetof(struct net_bridge_port_group, rhnode),
        .key_offset = offsetof(struct net_bridge_port_group, key),
        .key_len = sizeof(struct net_bridge_port_group_sg_key),
        .automatic_shrinking = true,
};

static void br_multicast_start_querier(struct net_bridge_mcast *brmctx,
                                       struct bridge_mcast_own_query *query);
static void br_ip4_multicast_add_router(struct net_bridge_mcast *brmctx,
                                        struct net_bridge_mcast_port *pmctx);
static void br_ip4_multicast_leave_group(struct net_bridge_mcast *brmctx,
                                         struct net_bridge_mcast_port *pmctx,
                                         __be32 group,
                                         __u16 vid,
                                         const unsigned char *src);
static void br_multicast_port_group_rexmit(struct timer_list *t);

static void
br_multicast_rport_del_notify(struct net_bridge_mcast_port *pmctx, bool deleted);
static void br_ip6_multicast_add_router(struct net_bridge_mcast *brmctx,
                                        struct net_bridge_mcast_port *pmctx);
#if IS_ENABLED(CONFIG_IPV6)
static void br_ip6_multicast_leave_group(struct net_bridge_mcast *brmctx,
                                         struct net_bridge_mcast_port *pmctx,
                                         const struct in6_addr *group,
                                         __u16 vid, const unsigned char *src);
#endif
static struct net_bridge_port_group *
__br_multicast_add_group(struct net_bridge_mcast *brmctx,
                         struct net_bridge_mcast_port *pmctx,
                         struct br_ip *group,
                         const unsigned char *src,
                         u8 filter_mode,
                         bool igmpv2_mldv1,
                         bool blocked);
static void br_multicast_find_del_pg(struct net_bridge *br,
                                     struct net_bridge_port_group *pg);
static void __br_multicast_stop(struct net_bridge_mcast *brmctx);

static int br_mc_disabled_update(struct net_device *dev, bool value,
                                 struct netlink_ext_ack *extack);

static struct net_bridge_port_group *
br_sg_port_find(struct net_bridge *br,
                struct net_bridge_port_group_sg_key *sg_p)
{
        lockdep_assert_held_once(&br->multicast_lock);

        return rhashtable_lookup_fast(&br->sg_port_tbl, sg_p,
                                      br_sg_port_rht_params);
}

static struct net_bridge_mdb_entry *br_mdb_ip_get_rcu(struct net_bridge *br,
                                                      struct br_ip *dst)
{
        return rhashtable_lookup(&br->mdb_hash_tbl, dst, br_mdb_rht_params);
}

struct net_bridge_mdb_entry *br_mdb_ip_get(struct net_bridge *br,
                                           struct br_ip *dst)
{
        struct net_bridge_mdb_entry *ent;

        lockdep_assert_held_once(&br->multicast_lock);

        rcu_read_lock();
        ent = rhashtable_lookup(&br->mdb_hash_tbl, dst, br_mdb_rht_params);
        rcu_read_unlock();

        return ent;
}

static struct net_bridge_mdb_entry *br_mdb_ip4_get(struct net_bridge *br,
                                                   __be32 dst, __u16 vid)
{
        struct br_ip br_dst;

        memset(&br_dst, 0, sizeof(br_dst));
        br_dst.dst.ip4 = dst;
        br_dst.proto = htons(ETH_P_IP);
        br_dst.vid = vid;

        return br_mdb_ip_get(br, &br_dst);
}

#if IS_ENABLED(CONFIG_IPV6)
static struct net_bridge_mdb_entry *br_mdb_ip6_get(struct net_bridge *br,
                                                   const struct in6_addr *dst,
                                                   __u16 vid)
{
        struct br_ip br_dst;

        memset(&br_dst, 0, sizeof(br_dst));
        br_dst.dst.ip6 = *dst;
        br_dst.proto = htons(ETH_P_IPV6);
        br_dst.vid = vid;

        return br_mdb_ip_get(br, &br_dst);
}
#endif

struct net_bridge_mdb_entry *
br_mdb_entry_skb_get(struct net_bridge_mcast *brmctx, struct sk_buff *skb,
                     u16 vid)
{
        struct net_bridge *br = brmctx->br;
        struct br_ip ip;

        if (!br_opt_get(br, BROPT_MULTICAST_ENABLED) ||
            br_multicast_ctx_vlan_global_disabled(brmctx))
                return NULL;

        if (BR_INPUT_SKB_CB(skb)->igmp)
                return NULL;

        memset(&ip, 0, sizeof(ip));
        ip.proto = skb->protocol;
        ip.vid = vid;

        switch (skb->protocol) {
        case htons(ETH_P_IP):
                ip.dst.ip4 = ip_hdr(skb)->daddr;
                if (brmctx->multicast_igmp_version == 3) {
                        struct net_bridge_mdb_entry *mdb;

                        ip.src.ip4 = ip_hdr(skb)->saddr;
                        mdb = br_mdb_ip_get_rcu(br, &ip);
                        if (mdb)
                                return mdb;
                        ip.src.ip4 = 0;
                }
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                ip.dst.ip6 = ipv6_hdr(skb)->daddr;
                if (brmctx->multicast_mld_version == 2) {
                        struct net_bridge_mdb_entry *mdb;

                        ip.src.ip6 = ipv6_hdr(skb)->saddr;
                        mdb = br_mdb_ip_get_rcu(br, &ip);
                        if (mdb)
                                return mdb;
                        memset(&ip.src.ip6, 0, sizeof(ip.src.ip6));
                }
                break;
#endif
        default:
                ip.proto = 0;
                ether_addr_copy(ip.dst.mac_addr, eth_hdr(skb)->h_dest);
        }

        return br_mdb_ip_get_rcu(br, &ip);
}

/* IMPORTANT: this function must be used only when the contexts cannot be
 * passed down (e.g. timer) and must be used for read-only purposes because
 * the vlan snooping option can change, so it can return any context
 * (non-vlan or vlan). Its initial intended purpose is to read timer values
 * from the *current* context based on the option. At worst that could lead
 * to inconsistent timers when the contexts are changed, i.e. src timer
 * which needs to re-arm with a specific delay taken from the old context
 */
static struct net_bridge_mcast_port *
br_multicast_pg_to_port_ctx(const struct net_bridge_port_group *pg)
{
        struct net_bridge_mcast_port *pmctx = &pg->key.port->multicast_ctx;
        struct net_bridge_vlan *vlan;

        lockdep_assert_held_once(&pg->key.port->br->multicast_lock);

        /* if vlan snooping is disabled use the port's multicast context */
        if (!pg->key.addr.vid ||
            !br_opt_get(pg->key.port->br, BROPT_MCAST_VLAN_SNOOPING_ENABLED))
                goto out;

        /* locking is tricky here, due to different rules for multicast and
         * vlans we need to take rcu to find the vlan and make sure it has
         * the BR_VLFLAG_MCAST_ENABLED flag set, it can only change under
         * multicast_lock which must be already held here, so the vlan's pmctx
         * can safely be used on return
         */
        rcu_read_lock();
        vlan = br_vlan_find(nbp_vlan_group_rcu(pg->key.port), pg->key.addr.vid);
        if (vlan && !br_multicast_port_ctx_vlan_disabled(&vlan->port_mcast_ctx))
                pmctx = &vlan->port_mcast_ctx;
        else
                pmctx = NULL;
        rcu_read_unlock();
out:
        return pmctx;
}

static struct net_bridge_mcast_port *
br_multicast_port_vid_to_port_ctx(struct net_bridge_port *port, u16 vid)
{
        struct net_bridge_mcast_port *pmctx = NULL;
        struct net_bridge_vlan *vlan;

        lockdep_assert_held_once(&port->br->multicast_lock);

        if (!br_opt_get(port->br, BROPT_MCAST_VLAN_SNOOPING_ENABLED))
                return NULL;

        /* Take RCU to access the vlan. */
        rcu_read_lock();

        vlan = br_vlan_find(nbp_vlan_group_rcu(port), vid);
        if (vlan && !br_multicast_port_ctx_vlan_disabled(&vlan->port_mcast_ctx))
                pmctx = &vlan->port_mcast_ctx;

        rcu_read_unlock();

        return pmctx;
}

/* when snooping we need to check if the contexts should be used
 * in the following order:
 * - if pmctx is non-NULL (port), check if it should be used
 * - if pmctx is NULL (bridge), check if brmctx should be used
 */
static bool
br_multicast_ctx_should_use(const struct net_bridge_mcast *brmctx,
                            const struct net_bridge_mcast_port *pmctx)
{
        if (!netif_running(brmctx->br->dev))
                return false;

        if (pmctx)
                return !br_multicast_port_ctx_state_disabled(pmctx);
        else
                return !br_multicast_ctx_vlan_disabled(brmctx);
}

static bool br_port_group_equal(struct net_bridge_port_group *p,
                                struct net_bridge_port *port,
                                const unsigned char *src)
{
        if (p->key.port != port)
                return false;

        if (!(port->flags & BR_MULTICAST_TO_UNICAST))
                return true;

        return ether_addr_equal(src, p->eth_addr);
}

static void __fwd_add_star_excl(struct net_bridge_mcast_port *pmctx,
                                struct net_bridge_port_group *pg,
                                struct br_ip *sg_ip)
{
        struct net_bridge_port_group_sg_key sg_key;
        struct net_bridge_port_group *src_pg;
        struct net_bridge_mcast *brmctx;

        memset(&sg_key, 0, sizeof(sg_key));
        brmctx = br_multicast_port_ctx_get_global(pmctx);
        sg_key.port = pg->key.port;
        sg_key.addr = *sg_ip;
        if (br_sg_port_find(brmctx->br, &sg_key))
                return;

        src_pg = __br_multicast_add_group(brmctx, pmctx,
                                          sg_ip, pg->eth_addr,
                                          MCAST_INCLUDE, false, false);
        if (IS_ERR_OR_NULL(src_pg) ||
            src_pg->rt_protocol != RTPROT_KERNEL)
                return;

        src_pg->flags |= MDB_PG_FLAGS_STAR_EXCL;
}

static void __fwd_del_star_excl(struct net_bridge_port_group *pg,
                                struct br_ip *sg_ip)
{
        struct net_bridge_port_group_sg_key sg_key;
        struct net_bridge *br = pg->key.port->br;
        struct net_bridge_port_group *src_pg;

        memset(&sg_key, 0, sizeof(sg_key));
        sg_key.port = pg->key.port;
        sg_key.addr = *sg_ip;
        src_pg = br_sg_port_find(br, &sg_key);
        if (!src_pg || !(src_pg->flags & MDB_PG_FLAGS_STAR_EXCL) ||
            src_pg->rt_protocol != RTPROT_KERNEL)
                return;

        br_multicast_find_del_pg(br, src_pg);
}

/* When a port group transitions to (or is added as) EXCLUDE we need to add it
 * to all other ports' S,G entries which are not blocked by the current group
 * for proper replication, the assumption is that any S,G blocked entries
 * are already added so the S,G,port lookup should skip them.
 * When a port group transitions from EXCLUDE -> INCLUDE mode or is being
 * deleted we need to remove it from all ports' S,G entries where it was
 * automatically installed before (i.e. where it's MDB_PG_FLAGS_STAR_EXCL).
 */
void br_multicast_star_g_handle_mode(struct net_bridge_port_group *pg,
                                     u8 filter_mode)
{
        struct net_bridge *br = pg->key.port->br;
        struct net_bridge_port_group *pg_lst;
        struct net_bridge_mcast_port *pmctx;
        struct net_bridge_mdb_entry *mp;
        struct br_ip sg_ip;

        if (WARN_ON(!br_multicast_is_star_g(&pg->key.addr)))
                return;

        mp = br_mdb_ip_get(br, &pg->key.addr);
        if (!mp)
                return;
        pmctx = br_multicast_pg_to_port_ctx(pg);
        if (!pmctx)
                return;

        memset(&sg_ip, 0, sizeof(sg_ip));
        sg_ip = pg->key.addr;

        for (pg_lst = mlock_dereference(mp->ports, br);
             pg_lst;
             pg_lst = mlock_dereference(pg_lst->next, br)) {
                struct net_bridge_group_src *src_ent;

                if (pg_lst == pg)
                        continue;
                hlist_for_each_entry(src_ent, &pg_lst->src_list, node) {
                        if (!(src_ent->flags & BR_SGRP_F_INSTALLED))
                                continue;
                        sg_ip.src = src_ent->addr.src;
                        switch (filter_mode) {
                        case MCAST_INCLUDE:
                                __fwd_del_star_excl(pg, &sg_ip);
                                break;
                        case MCAST_EXCLUDE:
                                __fwd_add_star_excl(pmctx, pg, &sg_ip);
                                break;
                        }
                }
        }
}

/* called when adding a new S,G with host_joined == false by default */
static void br_multicast_sg_host_state(struct net_bridge_mdb_entry *star_mp,
                                       struct net_bridge_port_group *sg)
{
        struct net_bridge_mdb_entry *sg_mp;

        if (WARN_ON(!br_multicast_is_star_g(&star_mp->addr)))
                return;
        if (!star_mp->host_joined)
                return;

        sg_mp = br_mdb_ip_get(star_mp->br, &sg->key.addr);
        if (!sg_mp)
                return;
        sg_mp->host_joined = true;
}

/* set the host_joined state of all of *,G's S,G entries */
static void br_multicast_star_g_host_state(struct net_bridge_mdb_entry *star_mp)
{
        struct net_bridge *br = star_mp->br;
        struct net_bridge_mdb_entry *sg_mp;
        struct net_bridge_port_group *pg;
        struct br_ip sg_ip;

        if (WARN_ON(!br_multicast_is_star_g(&star_mp->addr)))
                return;

        memset(&sg_ip, 0, sizeof(sg_ip));
        sg_ip = star_mp->addr;
        for (pg = mlock_dereference(star_mp->ports, br);
             pg;
             pg = mlock_dereference(pg->next, br)) {
                struct net_bridge_group_src *src_ent;

                hlist_for_each_entry(src_ent, &pg->src_list, node) {
                        if (!(src_ent->flags & BR_SGRP_F_INSTALLED))
                                continue;
                        sg_ip.src = src_ent->addr.src;
                        sg_mp = br_mdb_ip_get(br, &sg_ip);
                        if (!sg_mp)
                                continue;
                        sg_mp->host_joined = star_mp->host_joined;
                }
        }
}

static void br_multicast_sg_del_exclude_ports(struct net_bridge_mdb_entry *sgmp)
{
        struct net_bridge_port_group __rcu **pp;
        struct net_bridge_port_group *p;

        /* *,G exclude ports are only added to S,G entries */
        if (WARN_ON(br_multicast_is_star_g(&sgmp->addr)))
                return;

        /* we need the STAR_EXCLUDE ports if there are non-STAR_EXCLUDE ports
         * we should ignore perm entries since they're managed by user-space
         */
        for (pp = &sgmp->ports;
             (p = mlock_dereference(*pp, sgmp->br)) != NULL;
             pp = &p->next)
                if (!(p->flags & (MDB_PG_FLAGS_STAR_EXCL |
                                  MDB_PG_FLAGS_PERMANENT)))
                        return;

        /* currently the host can only have joined the *,G which means
         * we treat it as EXCLUDE {}, so for an S,G it's considered a
         * STAR_EXCLUDE entry and we can safely leave it
         */
        sgmp->host_joined = false;

        for (pp = &sgmp->ports;
             (p = mlock_dereference(*pp, sgmp->br)) != NULL;) {
                if (!(p->flags & MDB_PG_FLAGS_PERMANENT))
                        br_multicast_del_pg(sgmp, p, pp);
                else
                        pp = &p->next;
        }
}

void br_multicast_sg_add_exclude_ports(struct net_bridge_mdb_entry *star_mp,
                                       struct net_bridge_port_group *sg)
{
        struct net_bridge_port_group_sg_key sg_key;
        struct net_bridge *br = star_mp->br;
        struct net_bridge_mcast_port *pmctx;
        struct net_bridge_port_group *pg;
        struct net_bridge_mcast *brmctx;

        if (WARN_ON(br_multicast_is_star_g(&sg->key.addr)))
                return;
        if (WARN_ON(!br_multicast_is_star_g(&star_mp->addr)))
                return;

        br_multicast_sg_host_state(star_mp, sg);
        memset(&sg_key, 0, sizeof(sg_key));
        sg_key.addr = sg->key.addr;
        /* we need to add all exclude ports to the S,G */
        for (pg = mlock_dereference(star_mp->ports, br);
             pg;
             pg = mlock_dereference(pg->next, br)) {
                struct net_bridge_port_group *src_pg;

                if (pg == sg || pg->filter_mode == MCAST_INCLUDE)
                        continue;

                sg_key.port = pg->key.port;
                if (br_sg_port_find(br, &sg_key))
                        continue;

                pmctx = br_multicast_pg_to_port_ctx(pg);
                if (!pmctx)
                        continue;
                brmctx = br_multicast_port_ctx_get_global(pmctx);

                src_pg = __br_multicast_add_group(brmctx, pmctx,
                                                  &sg->key.addr,
                                                  sg->eth_addr,
                                                  MCAST_INCLUDE, false, false);
                if (IS_ERR_OR_NULL(src_pg) ||
                    src_pg->rt_protocol != RTPROT_KERNEL)
                        continue;
                src_pg->flags |= MDB_PG_FLAGS_STAR_EXCL;
        }
}

static void br_multicast_fwd_src_add(struct net_bridge_group_src *src)
{
        struct net_bridge_mdb_entry *star_mp;
        struct net_bridge_mcast_port *pmctx;
        struct net_bridge_port_group *sg;
        struct net_bridge_mcast *brmctx;
        struct br_ip sg_ip;

        if (src->flags & BR_SGRP_F_INSTALLED)
                return;

        memset(&sg_ip, 0, sizeof(sg_ip));
        pmctx = br_multicast_pg_to_port_ctx(src->pg);
        if (!pmctx)
                return;
        brmctx = br_multicast_port_ctx_get_global(pmctx);
        sg_ip = src->pg->key.addr;
        sg_ip.src = src->addr.src;

        sg = __br_multicast_add_group(brmctx, pmctx, &sg_ip,
                                      src->pg->eth_addr, MCAST_INCLUDE, false,
                                      !timer_pending(&src->timer));
        if (IS_ERR_OR_NULL(sg))
                return;
        src->flags |= BR_SGRP_F_INSTALLED;
        sg->flags &= ~MDB_PG_FLAGS_STAR_EXCL;

        /* if it was added by user-space as perm we can skip next steps */
        if (sg->rt_protocol != RTPROT_KERNEL &&
            (sg->flags & MDB_PG_FLAGS_PERMANENT))
                return;

        /* the kernel is now responsible for removing this S,G */
        del_timer(&sg->timer);
        star_mp = br_mdb_ip_get(src->br, &src->pg->key.addr);
        if (!star_mp)
                return;

        br_multicast_sg_add_exclude_ports(star_mp, sg);
}

static void br_multicast_fwd_src_remove(struct net_bridge_group_src *src,
                                        bool fastleave)
{
        struct net_bridge_port_group *p, *pg = src->pg;
        struct net_bridge_port_group __rcu **pp;
        struct net_bridge_mdb_entry *mp;
        struct br_ip sg_ip;

        memset(&sg_ip, 0, sizeof(sg_ip));
        sg_ip = pg->key.addr;
        sg_ip.src = src->addr.src;

        mp = br_mdb_ip_get(src->br, &sg_ip);
        if (!mp)
                return;

        for (pp = &mp->ports;
             (p = mlock_dereference(*pp, src->br)) != NULL;
             pp = &p->next) {
                if (!br_port_group_equal(p, pg->key.port, pg->eth_addr))
                        continue;

                if (p->rt_protocol != RTPROT_KERNEL &&
                    (p->flags & MDB_PG_FLAGS_PERMANENT) &&
                    !(src->flags & BR_SGRP_F_USER_ADDED))
                        break;

                if (fastleave)
                        p->flags |= MDB_PG_FLAGS_FAST_LEAVE;
                br_multicast_del_pg(mp, p, pp);
                break;
        }
        src->flags &= ~BR_SGRP_F_INSTALLED;
}

/* install S,G and based on src's timer enable or disable forwarding */
static void br_multicast_fwd_src_handle(struct net_bridge_group_src *src)
{
        struct net_bridge_port_group_sg_key sg_key;
        struct net_bridge_port_group *sg;
        u8 old_flags;

        br_multicast_fwd_src_add(src);

        memset(&sg_key, 0, sizeof(sg_key));
        sg_key.addr = src->pg->key.addr;
        sg_key.addr.src = src->addr.src;
        sg_key.port = src->pg->key.port;

        sg = br_sg_port_find(src->br, &sg_key);
        if (!sg || (sg->flags & MDB_PG_FLAGS_PERMANENT))
                return;

        old_flags = sg->flags;
        if (timer_pending(&src->timer))
                sg->flags &= ~MDB_PG_FLAGS_BLOCKED;
        else
                sg->flags |= MDB_PG_FLAGS_BLOCKED;

        if (old_flags != sg->flags) {
                struct net_bridge_mdb_entry *sg_mp;

                sg_mp = br_mdb_ip_get(src->br, &sg_key.addr);
                if (!sg_mp)
                        return;
                br_mdb_notify(src->br->dev, sg_mp, sg, RTM_NEWMDB);
        }
}

static void br_multicast_destroy_mdb_entry(struct net_bridge_mcast_gc *gc)
{
        struct net_bridge_mdb_entry *mp;

        mp = container_of(gc, struct net_bridge_mdb_entry, mcast_gc);
        WARN_ON(!hlist_unhashed(&mp->mdb_node));
        WARN_ON(mp->ports);

        timer_shutdown_sync(&mp->timer);
        kfree_rcu(mp, rcu);
}

static void br_multicast_del_mdb_entry(struct net_bridge_mdb_entry *mp)
{
        struct net_bridge *br = mp->br;

        rhashtable_remove_fast(&br->mdb_hash_tbl, &mp->rhnode,
                               br_mdb_rht_params);
        hlist_del_init_rcu(&mp->mdb_node);
        hlist_add_head(&mp->mcast_gc.gc_node, &br->mcast_gc_list);
        queue_work(system_long_wq, &br->mcast_gc_work);
}

static void br_multicast_group_expired(struct timer_list *t)
{
        struct net_bridge_mdb_entry *mp = from_timer(mp, t, timer);
        struct net_bridge *br = mp->br;

        spin_lock(&br->multicast_lock);
        if (hlist_unhashed(&mp->mdb_node) || !netif_running(br->dev) ||
            timer_pending(&mp->timer))
                goto out;

        br_multicast_host_leave(mp, true);

        if (mp->ports)
                goto out;
        br_multicast_del_mdb_entry(mp);
out:
        spin_unlock(&br->multicast_lock);
}

static void br_multicast_destroy_group_src(struct net_bridge_mcast_gc *gc)
{
        struct net_bridge_group_src *src;

        src = container_of(gc, struct net_bridge_group_src, mcast_gc);
        WARN_ON(!hlist_unhashed(&src->node));

        timer_shutdown_sync(&src->timer);
        kfree_rcu(src, rcu);
}

void __br_multicast_del_group_src(struct net_bridge_group_src *src)
{
        struct net_bridge *br = src->pg->key.port->br;

        hlist_del_init_rcu(&src->node);
        src->pg->src_ents--;
        hlist_add_head(&src->mcast_gc.gc_node, &br->mcast_gc_list);
        queue_work(system_long_wq, &br->mcast_gc_work);
}

void br_multicast_del_group_src(struct net_bridge_group_src *src,
                                bool fastleave)
{
        br_multicast_fwd_src_remove(src, fastleave);
        __br_multicast_del_group_src(src);
}

static int
br_multicast_port_ngroups_inc_one(struct net_bridge_mcast_port *pmctx,
                                  struct netlink_ext_ack *extack,
                                  const char *what)
{
        u32 max = READ_ONCE(pmctx->mdb_max_entries);
        u32 n = READ_ONCE(pmctx->mdb_n_entries);

        if (max && n >= max) {
                NL_SET_ERR_MSG_FMT_MOD(extack, "%s is already in %u groups, and mcast_max_groups=%u",
                                       what, n, max);
                return -E2BIG;
        }

        WRITE_ONCE(pmctx->mdb_n_entries, n + 1);
        return 0;
}

static void br_multicast_port_ngroups_dec_one(struct net_bridge_mcast_port *pmctx)
{
        u32 n = READ_ONCE(pmctx->mdb_n_entries);

        WARN_ON_ONCE(n == 0);
        WRITE_ONCE(pmctx->mdb_n_entries, n - 1);
}

static int br_multicast_port_ngroups_inc(struct net_bridge_port *port,
                                         const struct br_ip *group,
                                         struct netlink_ext_ack *extack)
{
        struct net_bridge_mcast_port *pmctx;
        int err;

        lockdep_assert_held_once(&port->br->multicast_lock);

        /* Always count on the port context. */
        err = br_multicast_port_ngroups_inc_one(&port->multicast_ctx, extack,
                                                "Port");
        if (err) {
                trace_br_mdb_full(port->dev, group);
                return err;
        }

        /* Only count on the VLAN context if VID is given, and if snooping on
         * that VLAN is enabled.
         */
        if (!group->vid)
                return 0;

        pmctx = br_multicast_port_vid_to_port_ctx(port, group->vid);
        if (!pmctx)
                return 0;

        err = br_multicast_port_ngroups_inc_one(pmctx, extack, "Port-VLAN");
        if (err) {
                trace_br_mdb_full(port->dev, group);
                goto dec_one_out;
        }

        return 0;

dec_one_out:
        br_multicast_port_ngroups_dec_one(&port->multicast_ctx);
        return err;
}

static void br_multicast_port_ngroups_dec(struct net_bridge_port *port, u16 vid)
{
        struct net_bridge_mcast_port *pmctx;

        lockdep_assert_held_once(&port->br->multicast_lock);

        if (vid) {
                pmctx = br_multicast_port_vid_to_port_ctx(port, vid);
                if (pmctx)
                        br_multicast_port_ngroups_dec_one(pmctx);
        }
        br_multicast_port_ngroups_dec_one(&port->multicast_ctx);
}

u32 br_multicast_ngroups_get(const struct net_bridge_mcast_port *pmctx)
{
        return READ_ONCE(pmctx->mdb_n_entries);
}

void br_multicast_ngroups_set_max(struct net_bridge_mcast_port *pmctx, u32 max)
{
        WRITE_ONCE(pmctx->mdb_max_entries, max);
}

u32 br_multicast_ngroups_get_max(const struct net_bridge_mcast_port *pmctx)
{
        return READ_ONCE(pmctx->mdb_max_entries);
}

static void br_multicast_destroy_port_group(struct net_bridge_mcast_gc *gc)
{
        struct net_bridge_port_group *pg;

        pg = container_of(gc, struct net_bridge_port_group, mcast_gc);
        WARN_ON(!hlist_unhashed(&pg->mglist));
        WARN_ON(!hlist_empty(&pg->src_list));

        timer_shutdown_sync(&pg->rexmit_timer);
        timer_shutdown_sync(&pg->timer);
        kfree_rcu(pg, rcu);
}

void br_multicast_del_pg(struct net_bridge_mdb_entry *mp,
                         struct net_bridge_port_group *pg,
                         struct net_bridge_port_group __rcu **pp)
{
        struct net_bridge *br = pg->key.port->br;
        struct net_bridge_group_src *ent;
        struct hlist_node *tmp;

        rcu_assign_pointer(*pp, pg->next);
        hlist_del_init(&pg->mglist);
        br_multicast_eht_clean_sets(pg);
        hlist_for_each_entry_safe(ent, tmp, &pg->src_list, node)
                br_multicast_del_group_src(ent, false);
        br_mdb_notify(br->dev, mp, pg, RTM_DELMDB);
        if (!br_multicast_is_star_g(&mp->addr)) {
                rhashtable_remove_fast(&br->sg_port_tbl, &pg->rhnode,
                                       br_sg_port_rht_params);
                br_multicast_sg_del_exclude_ports(mp);
        } else {
                br_multicast_star_g_handle_mode(pg, MCAST_INCLUDE);
        }
        br_multicast_port_ngroups_dec(pg->key.port, pg->key.addr.vid);
        hlist_add_head(&pg->mcast_gc.gc_node, &br->mcast_gc_list);
        queue_work(system_long_wq, &br->mcast_gc_work);

        if (!mp->ports && !mp->host_joined && netif_running(br->dev))
                mod_timer(&mp->timer, jiffies);
}

static void br_multicast_find_del_pg(struct net_bridge *br,
                                     struct net_bridge_port_group *pg)
{
        struct net_bridge_port_group __rcu **pp;
        struct net_bridge_mdb_entry *mp;
        struct net_bridge_port_group *p;

        mp = br_mdb_ip_get(br, &pg->key.addr);
        if (WARN_ON(!mp))
                return;

        for (pp = &mp->ports;
             (p = mlock_dereference(*pp, br)) != NULL;
             pp = &p->next) {
                if (p != pg)
                        continue;

                br_multicast_del_pg(mp, pg, pp);
                return;
        }

        WARN_ON(1);
}

static void br_multicast_port_group_expired(struct timer_list *t)
{
        struct net_bridge_port_group *pg = from_timer(pg, t, timer);
        struct net_bridge_group_src *src_ent;
        struct net_bridge *br = pg->key.port->br;
        struct hlist_node *tmp;
        bool changed;

        spin_lock(&br->multicast_lock);
        if (!netif_running(br->dev) || timer_pending(&pg->timer) ||
            hlist_unhashed(&pg->mglist) || pg->flags & MDB_PG_FLAGS_PERMANENT)
                goto out;

        changed = !!(pg->filter_mode == MCAST_EXCLUDE);
        pg->filter_mode = MCAST_INCLUDE;
        hlist_for_each_entry_safe(src_ent, tmp, &pg->src_list, node) {
                if (!timer_pending(&src_ent->timer)) {
                        br_multicast_del_group_src(src_ent, false);
                        changed = true;
                }
        }

        if (hlist_empty(&pg->src_list)) {
                br_multicast_find_del_pg(br, pg);
        } else if (changed) {
                struct net_bridge_mdb_entry *mp = br_mdb_ip_get(br, &pg->key.addr);

                if (changed && br_multicast_is_star_g(&pg->key.addr))
                        br_multicast_star_g_handle_mode(pg, MCAST_INCLUDE);

                if (WARN_ON(!mp))
                        goto out;
                br_mdb_notify(br->dev, mp, pg, RTM_NEWMDB);
        }
out:
        spin_unlock(&br->multicast_lock);
}

static void br_multicast_gc(struct hlist_head *head)
{
        struct net_bridge_mcast_gc *gcent;
        struct hlist_node *tmp;

        hlist_for_each_entry_safe(gcent, tmp, head, gc_node) {
                hlist_del_init(&gcent->gc_node);
                gcent->destroy(gcent);
        }
}

static void __br_multicast_query_handle_vlan(struct net_bridge_mcast *brmctx,
                                             struct net_bridge_mcast_port *pmctx,
                                             struct sk_buff *skb)
{
        struct net_bridge_vlan *vlan = NULL;

        if (pmctx && br_multicast_port_ctx_is_vlan(pmctx))
                vlan = pmctx->vlan;
        else if (br_multicast_ctx_is_vlan(brmctx))
                vlan = brmctx->vlan;

        if (vlan && !(vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED)) {
                u16 vlan_proto;

                if (br_vlan_get_proto(brmctx->br->dev, &vlan_proto) != 0)
                        return;
                __vlan_hwaccel_put_tag(skb, htons(vlan_proto), vlan->vid);
        }
}

static struct sk_buff *br_ip4_multicast_alloc_query(struct net_bridge_mcast *brmctx,
                                                    struct net_bridge_mcast_port *pmctx,
                                                    struct net_bridge_port_group *pg,
                                                    __be32 ip_dst, __be32 group,
                                                    bool with_srcs, bool over_lmqt,
                                                    u8 sflag, u8 *igmp_type,
                                                    bool *need_rexmit)
{
        struct net_bridge_port *p = pg ? pg->key.port : NULL;
        struct net_bridge_group_src *ent;
        size_t pkt_size, igmp_hdr_size;
        unsigned long now = jiffies;
        struct igmpv3_query *ihv3;
        void *csum_start = NULL;
        __sum16 *csum = NULL;
        struct sk_buff *skb;
        struct igmphdr *ih;
        struct ethhdr *eth;
        unsigned long lmqt;
        struct iphdr *iph;
        u16 lmqt_srcs = 0;

        igmp_hdr_size = sizeof(*ih);
        if (brmctx->multicast_igmp_version == 3) {
                igmp_hdr_size = sizeof(*ihv3);
                if (pg && with_srcs) {
                        lmqt = now + (brmctx->multicast_last_member_interval *
                                      brmctx->multicast_last_member_count);
                        hlist_for_each_entry(ent, &pg->src_list, node) {
                                if (over_lmqt == time_after(ent->timer.expires,
                                                            lmqt) &&
                                    ent->src_query_rexmit_cnt > 0)
                                        lmqt_srcs++;
                        }

                        if (!lmqt_srcs)
                                return NULL;
                        igmp_hdr_size += lmqt_srcs * sizeof(__be32);
                }
        }

        pkt_size = sizeof(*eth) + sizeof(*iph) + 4 + igmp_hdr_size;
        if ((p && pkt_size > p->dev->mtu) ||
            pkt_size > brmctx->br->dev->mtu)
                return NULL;

        skb = netdev_alloc_skb_ip_align(brmctx->br->dev, pkt_size);
        if (!skb)
                goto out;

        __br_multicast_query_handle_vlan(brmctx, pmctx, skb);
        skb->protocol = htons(ETH_P_IP);

        skb_reset_mac_header(skb);
        eth = eth_hdr(skb);

        ether_addr_copy(eth->h_source, brmctx->br->dev->dev_addr);
        ip_eth_mc_map(ip_dst, eth->h_dest);
        eth->h_proto = htons(ETH_P_IP);
        skb_put(skb, sizeof(*eth));

        skb_set_network_header(skb, skb->len);
        iph = ip_hdr(skb);
        iph->tot_len = htons(pkt_size - sizeof(*eth));

        iph->version = 4;
        iph->ihl = 6;
        iph->tos = 0xc0;
        iph->id = 0;
        iph->frag_off = htons(IP_DF);
        iph->ttl = 1;
        iph->protocol = IPPROTO_IGMP;
        iph->saddr = br_opt_get(brmctx->br, BROPT_MULTICAST_QUERY_USE_IFADDR) ?
                     inet_select_addr(brmctx->br->dev, 0, RT_SCOPE_LINK) : 0;
        iph->daddr = ip_dst;
        ((u8 *)&iph[1])[0] = IPOPT_RA;
        ((u8 *)&iph[1])[1] = 4;
        ((u8 *)&iph[1])[2] = 0;
        ((u8 *)&iph[1])[3] = 0;
        ip_send_check(iph);
        skb_put(skb, 24);

        skb_set_transport_header(skb, skb->len);
        *igmp_type = IGMP_HOST_MEMBERSHIP_QUERY;

        switch (brmctx->multicast_igmp_version) {
        case 2:
                ih = igmp_hdr(skb);
                ih->type = IGMP_HOST_MEMBERSHIP_QUERY;
                ih->code = (group ? brmctx->multicast_last_member_interval :
                                    brmctx->multicast_query_response_interval) /
                           (HZ / IGMP_TIMER_SCALE);
                ih->group = group;
                ih->csum = 0;
                csum = &ih->csum;
                csum_start = (void *)ih;
                break;
        case 3:
                ihv3 = igmpv3_query_hdr(skb);
                ihv3->type = IGMP_HOST_MEMBERSHIP_QUERY;
                ihv3->code = (group ? brmctx->multicast_last_member_interval :
                                      brmctx->multicast_query_response_interval) /
                             (HZ / IGMP_TIMER_SCALE);
                ihv3->group = group;
                ihv3->qqic = brmctx->multicast_query_interval / HZ;
                ihv3->nsrcs = htons(lmqt_srcs);
                ihv3->resv = 0;
                ihv3->suppress = sflag;
                ihv3->qrv = 2;
                ihv3->csum = 0;
                csum = &ihv3->csum;
                csum_start = (void *)ihv3;
                if (!pg || !with_srcs)
                        break;

                lmqt_srcs = 0;
                hlist_for_each_entry(ent, &pg->src_list, node) {
                        if (over_lmqt == time_after(ent->timer.expires,
                                                    lmqt) &&
                            ent->src_query_rexmit_cnt > 0) {
                                ihv3->srcs[lmqt_srcs++] = ent->addr.src.ip4;
                                ent->src_query_rexmit_cnt--;
                                if (need_rexmit && ent->src_query_rexmit_cnt)
                                        *need_rexmit = true;
                        }
                }
                if (WARN_ON(lmqt_srcs != ntohs(ihv3->nsrcs))) {
                        kfree_skb(skb);
                        return NULL;
                }
                break;
        }

        if (WARN_ON(!csum || !csum_start)) {
                kfree_skb(skb);
                return NULL;
        }

        *csum = ip_compute_csum(csum_start, igmp_hdr_size);
        skb_put(skb, igmp_hdr_size);
        __skb_pull(skb, sizeof(*eth));

out:
        return skb;
}

#if IS_ENABLED(CONFIG_IPV6)
static struct sk_buff *br_ip6_multicast_alloc_query(struct net_bridge_mcast *brmctx,
                                                    struct net_bridge_mcast_port *pmctx,
                                                    struct net_bridge_port_group *pg,
                                                    const struct in6_addr *ip6_dst,
                                                    const struct in6_addr *group,
                                                    bool with_srcs, bool over_llqt,
                                                    u8 sflag, u8 *igmp_type,
                                                    bool *need_rexmit)
{
        struct net_bridge_port *p = pg ? pg->key.port : NULL;
        struct net_bridge_group_src *ent;
        size_t pkt_size, mld_hdr_size;
        unsigned long now = jiffies;
        struct mld2_query *mld2q;
        void *csum_start = NULL;
        unsigned long interval;
        __sum16 *csum = NULL;
        struct ipv6hdr *ip6h;
        struct mld_msg *mldq;
        struct sk_buff *skb;
        unsigned long llqt;
        struct ethhdr *eth;
        u16 llqt_srcs = 0;
        u8 *hopopt;

        mld_hdr_size = sizeof(*mldq);
        if (brmctx->multicast_mld_version == 2) {
                mld_hdr_size = sizeof(*mld2q);
                if (pg && with_srcs) {
                        llqt = now + (brmctx->multicast_last_member_interval *
                                      brmctx->multicast_last_member_count);
                        hlist_for_each_entry(ent, &pg->src_list, node) {
                                if (over_llqt == time_after(ent->timer.expires,
                                                            llqt) &&
                                    ent->src_query_rexmit_cnt > 0)
                                        llqt_srcs++;
                        }

                        if (!llqt_srcs)
                                return NULL;
                        mld_hdr_size += llqt_srcs * sizeof(struct in6_addr);
                }
        }

        pkt_size = sizeof(*eth) + sizeof(*ip6h) + 8 + mld_hdr_size;
        if ((p && pkt_size > p->dev->mtu) ||
            pkt_size > brmctx->br->dev->mtu)
                return NULL;

        skb = netdev_alloc_skb_ip_align(brmctx->br->dev, pkt_size);
        if (!skb)
                goto out;

        __br_multicast_query_handle_vlan(brmctx, pmctx, skb);
        skb->protocol = htons(ETH_P_IPV6);

        /* Ethernet header */
        skb_reset_mac_header(skb);
        eth = eth_hdr(skb);

        ether_addr_copy(eth->h_source, brmctx->br->dev->dev_addr);
        eth->h_proto = htons(ETH_P_IPV6);
        skb_put(skb, sizeof(*eth));

        /* IPv6 header + HbH option */
        skb_set_network_header(skb, skb->len);
        ip6h = ipv6_hdr(skb);

        *(__force __be32 *)ip6h = htonl(0x60000000);
        ip6h->payload_len = htons(8 + mld_hdr_size);
        ip6h->nexthdr = IPPROTO_HOPOPTS;
        ip6h->hop_limit = 1;
        ip6h->daddr = *ip6_dst;
        if (ipv6_dev_get_saddr(dev_net(brmctx->br->dev), brmctx->br->dev,
                               &ip6h->daddr, 0, &ip6h->saddr)) {
                kfree_skb(skb);
                br_opt_toggle(brmctx->br, BROPT_HAS_IPV6_ADDR, false);
                return NULL;
        }

        br_opt_toggle(brmctx->br, BROPT_HAS_IPV6_ADDR, true);
        ipv6_eth_mc_map(&ip6h->daddr, eth->h_dest);

        hopopt = (u8 *)(ip6h + 1);
        hopopt[0] = IPPROTO_ICMPV6;                /* next hdr */
        hopopt[1] = 0;                                /* length of HbH */
        hopopt[2] = IPV6_TLV_ROUTERALERT;        /* Router Alert */
        hopopt[3] = 2;                                /* Length of RA Option */
        hopopt[4] = 0;                                /* Type = 0x0000 (MLD) */
        hopopt[5] = 0;
        hopopt[6] = IPV6_TLV_PAD1;                /* Pad1 */
        hopopt[7] = IPV6_TLV_PAD1;                /* Pad1 */

        skb_put(skb, sizeof(*ip6h) + 8);

        /* ICMPv6 */
        skb_set_transport_header(skb, skb->len);
        interval = ipv6_addr_any(group) ?
                        brmctx->multicast_query_response_interval :
                        brmctx->multicast_last_member_interval;
        *igmp_type = ICMPV6_MGM_QUERY;
        switch (brmctx->multicast_mld_version) {
        case 1:
                mldq = (struct mld_msg *)icmp6_hdr(skb);
                mldq->mld_type = ICMPV6_MGM_QUERY;
                mldq->mld_code = 0;
                mldq->mld_cksum = 0;
                mldq->mld_maxdelay = htons((u16)jiffies_to_msecs(interval));
                mldq->mld_reserved = 0;
                mldq->mld_mca = *group;
                csum = &mldq->mld_cksum;
                csum_start = (void *)mldq;
                break;
        case 2:
                mld2q = (struct mld2_query *)icmp6_hdr(skb);
                mld2q->mld2q_mrc = htons((u16)jiffies_to_msecs(interval));
                mld2q->mld2q_type = ICMPV6_MGM_QUERY;
                mld2q->mld2q_code = 0;
                mld2q->mld2q_cksum = 0;
                mld2q->mld2q_resv1 = 0;
                mld2q->mld2q_resv2 = 0;
                mld2q->mld2q_suppress = sflag;
                mld2q->mld2q_qrv = 2;
                mld2q->mld2q_nsrcs = htons(llqt_srcs);
                mld2q->mld2q_qqic = brmctx->multicast_query_interval / HZ;
                mld2q->mld2q_mca = *group;
                csum = &mld2q->mld2q_cksum;
                csum_start = (void *)mld2q;
                if (!pg || !with_srcs)
                        break;

                llqt_srcs = 0;
                hlist_for_each_entry(ent, &pg->src_list, node) {
                        if (over_llqt == time_after(ent->timer.expires,
                                                    llqt) &&
                            ent->src_query_rexmit_cnt > 0) {
                                mld2q->mld2q_srcs[llqt_srcs++] = ent->addr.src.ip6;
                                ent->src_query_rexmit_cnt--;
                                if (need_rexmit && ent->src_query_rexmit_cnt)
                                        *need_rexmit = true;
                        }
                }
                if (WARN_ON(llqt_srcs != ntohs(mld2q->mld2q_nsrcs))) {
                        kfree_skb(skb);
                        return NULL;
                }
                break;
        }

        if (WARN_ON(!csum || !csum_start)) {
                kfree_skb(skb);
                return NULL;
        }

        *csum = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr, mld_hdr_size,
                                IPPROTO_ICMPV6,
                                csum_partial(csum_start, mld_hdr_size, 0));
        skb_put(skb, mld_hdr_size);
        __skb_pull(skb, sizeof(*eth));

out:
        return skb;
}
#endif

static struct sk_buff *br_multicast_alloc_query(struct net_bridge_mcast *brmctx,
                                                struct net_bridge_mcast_port *pmctx,
                                                struct net_bridge_port_group *pg,
                                                struct br_ip *ip_dst,
                                                struct br_ip *group,
                                                bool with_srcs, bool over_lmqt,
                                                u8 sflag, u8 *igmp_type,
                                                bool *need_rexmit)
{
        __be32 ip4_dst;

        switch (group->proto) {
        case htons(ETH_P_IP):
                ip4_dst = ip_dst ? ip_dst->dst.ip4 : htonl(INADDR_ALLHOSTS_GROUP);
                return br_ip4_multicast_alloc_query(brmctx, pmctx, pg,
                                                    ip4_dst, group->dst.ip4,
                                                    with_srcs, over_lmqt,
                                                    sflag, igmp_type,
                                                    need_rexmit);
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6): {
                struct in6_addr ip6_dst;

                if (ip_dst)
                        ip6_dst = ip_dst->dst.ip6;
                else
                        ipv6_addr_set(&ip6_dst, htonl(0xff020000), 0, 0,
                                      htonl(1));

                return br_ip6_multicast_alloc_query(brmctx, pmctx, pg,
                                                    &ip6_dst, &group->dst.ip6,
                                                    with_srcs, over_lmqt,
                                                    sflag, igmp_type,
                                                    need_rexmit);
        }
#endif
        }
        return NULL;
}

struct net_bridge_mdb_entry *br_multicast_new_group(struct net_bridge *br,
                                                    struct br_ip *group)
{
        struct net_bridge_mdb_entry *mp;
        int err;

        mp = br_mdb_ip_get(br, group);
        if (mp)
                return mp;

        if (atomic_read(&br->mdb_hash_tbl.nelems) >= br->hash_max) {
                trace_br_mdb_full(br->dev, group);
                br_mc_disabled_update(br->dev, false, NULL);
                br_opt_toggle(br, BROPT_MULTICAST_ENABLED, false);
                return ERR_PTR(-E2BIG);
        }

        mp = kzalloc(sizeof(*mp), GFP_ATOMIC);
        if (unlikely(!mp))
                return ERR_PTR(-ENOMEM);

        mp->br = br;
        mp->addr = *group;
        mp->mcast_gc.destroy = br_multicast_destroy_mdb_entry;
        timer_setup(&mp->timer, br_multicast_group_expired, 0);
        err = rhashtable_lookup_insert_fast(&br->mdb_hash_tbl, &mp->rhnode,
                                            br_mdb_rht_params);
        if (err) {
                kfree(mp);
                mp = ERR_PTR(err);
        } else {
                hlist_add_head_rcu(&mp->mdb_node, &br->mdb_list);
        }

        return mp;
}

static void br_multicast_group_src_expired(struct timer_list *t)
{
        struct net_bridge_group_src *src = from_timer(src, t, timer);
        struct net_bridge_port_group *pg;
        struct net_bridge *br = src->br;

        spin_lock(&br->multicast_lock);
        if (hlist_unhashed(&src->node) || !netif_running(br->dev) ||
            timer_pending(&src->timer))
                goto out;

        pg = src->pg;
        if (pg->filter_mode == MCAST_INCLUDE) {
                br_multicast_del_group_src(src, false);
                if (!hlist_empty(&pg->src_list))
                        goto out;
                br_multicast_find_del_pg(br, pg);
        } else {
                br_multicast_fwd_src_handle(src);
        }

out:
        spin_unlock(&br->multicast_lock);
}

struct net_bridge_group_src *
br_multicast_find_group_src(struct net_bridge_port_group *pg, struct br_ip *ip)
{
        struct net_bridge_group_src *ent;

        switch (ip->proto) {
        case htons(ETH_P_IP):
                hlist_for_each_entry(ent, &pg->src_list, node)
                        if (ip->src.ip4 == ent->addr.src.ip4)
                                return ent;
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                hlist_for_each_entry(ent, &pg->src_list, node)
                        if (!ipv6_addr_cmp(&ent->addr.src.ip6, &ip->src.ip6))
                                return ent;
                break;
#endif
        }

        return NULL;
}

struct net_bridge_group_src *
br_multicast_new_group_src(struct net_bridge_port_group *pg, struct br_ip *src_ip)
{
        struct net_bridge_group_src *grp_src;

        if (unlikely(pg->src_ents >= PG_SRC_ENT_LIMIT))
                return NULL;

        switch (src_ip->proto) {
        case htons(ETH_P_IP):
                if (ipv4_is_zeronet(src_ip->src.ip4) ||
                    ipv4_is_multicast(src_ip->src.ip4))
                        return NULL;
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                if (ipv6_addr_any(&src_ip->src.ip6) ||
                    ipv6_addr_is_multicast(&src_ip->src.ip6))
                        return NULL;
                break;
#endif
        }

        grp_src = kzalloc(sizeof(*grp_src), GFP_ATOMIC);
        if (unlikely(!grp_src))
                return NULL;

        grp_src->pg = pg;
        grp_src->br = pg->key.port->br;
        grp_src->addr = *src_ip;
        grp_src->mcast_gc.destroy = br_multicast_destroy_group_src;
        timer_setup(&grp_src->timer, br_multicast_group_src_expired, 0);

        hlist_add_head_rcu(&grp_src->node, &pg->src_list);
        pg->src_ents++;

        return grp_src;
}

struct net_bridge_port_group *br_multicast_new_port_group(
                        struct net_bridge_port *port,
                        const struct br_ip *group,
                        struct net_bridge_port_group __rcu *next,
                        unsigned char flags,
                        const unsigned char *src,
                        u8 filter_mode,
                        u8 rt_protocol,
                        struct netlink_ext_ack *extack)
{
        struct net_bridge_port_group *p;
        int err;

        err = br_multicast_port_ngroups_inc(port, group, extack);
        if (err)
                return NULL;

        p = kzalloc(sizeof(*p), GFP_ATOMIC);
        if (unlikely(!p)) {
                NL_SET_ERR_MSG_MOD(extack, "Couldn't allocate new port group");
                goto dec_out;
        }

        p->key.addr = *group;
        p->key.port = port;
        p->flags = flags;
        p->filter_mode = filter_mode;
        p->rt_protocol = rt_protocol;
        p->eht_host_tree = RB_ROOT;
        p->eht_set_tree = RB_ROOT;
        p->mcast_gc.destroy = br_multicast_destroy_port_group;
        INIT_HLIST_HEAD(&p->src_list);

        if (!br_multicast_is_star_g(group) &&
            rhashtable_lookup_insert_fast(&port->br->sg_port_tbl, &p->rhnode,
                                          br_sg_port_rht_params)) {
                NL_SET_ERR_MSG_MOD(extack, "Couldn't insert new port group");
                goto free_out;
        }

        rcu_assign_pointer(p->next, next);
        timer_setup(&p->timer, br_multicast_port_group_expired, 0);
        timer_setup(&p->rexmit_timer, br_multicast_port_group_rexmit, 0);
        hlist_add_head(&p->mglist, &port->mglist);

        if (src)
                memcpy(p->eth_addr, src, ETH_ALEN);
        else
                eth_broadcast_addr(p->eth_addr);

        return p;

free_out:
        kfree(p);
dec_out:
        br_multicast_port_ngroups_dec(port, group->vid);
        return NULL;
}

void br_multicast_del_port_group(struct net_bridge_port_group *p)
{
        struct net_bridge_port *port = p->key.port;
        __u16 vid = p->key.addr.vid;

        hlist_del_init(&p->mglist);
        if (!br_multicast_is_star_g(&p->key.addr))
                rhashtable_remove_fast(&port->br->sg_port_tbl, &p->rhnode,
                                       br_sg_port_rht_params);
        kfree(p);
        br_multicast_port_ngroups_dec(port, vid);
}

void br_multicast_host_join(const struct net_bridge_mcast *brmctx,
                            struct net_bridge_mdb_entry *mp, bool notify)
{
        if (!mp->host_joined) {
                mp->host_joined = true;
                if (br_multicast_is_star_g(&mp->addr))
                        br_multicast_star_g_host_state(mp);
                if (notify)
                        br_mdb_notify(mp->br->dev, mp, NULL, RTM_NEWMDB);
        }

        if (br_group_is_l2(&mp->addr))
                return;

        mod_timer(&mp->timer, jiffies + brmctx->multicast_membership_interval);
}

void br_multicast_host_leave(struct net_bridge_mdb_entry *mp, bool notify)
{
        if (!mp->host_joined)
                return;

        mp->host_joined = false;
        if (br_multicast_is_star_g(&mp->addr))
                br_multicast_star_g_host_state(mp);
        if (notify)
                br_mdb_notify(mp->br->dev, mp, NULL, RTM_DELMDB);
}

static struct net_bridge_port_group *
__br_multicast_add_group(struct net_bridge_mcast *brmctx,
                         struct net_bridge_mcast_port *pmctx,
                         struct br_ip *group,
                         const unsigned char *src,
                         u8 filter_mode,
                         bool igmpv2_mldv1,
                         bool blocked)
{
        struct net_bridge_port_group __rcu **pp;
        struct net_bridge_port_group *p = NULL;
        struct net_bridge_mdb_entry *mp;
        unsigned long now = jiffies;

        if (!br_multicast_ctx_should_use(brmctx, pmctx))
                goto out;

        mp = br_multicast_new_group(brmctx->br, group);
        if (IS_ERR(mp))
                return ERR_CAST(mp);

        if (!pmctx) {
                br_multicast_host_join(brmctx, mp, true);
                goto out;
        }

        for (pp = &mp->ports;
             (p = mlock_dereference(*pp, brmctx->br)) != NULL;
             pp = &p->next) {
                if (br_port_group_equal(p, pmctx->port, src))
                        goto found;
                if ((unsigned long)p->key.port < (unsigned long)pmctx->port)
                        break;
        }

        p = br_multicast_new_port_group(pmctx->port, group, *pp, 0, src,
                                        filter_mode, RTPROT_KERNEL, NULL);
        if (unlikely(!p)) {
                p = ERR_PTR(-ENOMEM);
                goto out;
        }
        rcu_assign_pointer(*pp, p);
        if (blocked)
                p->flags |= MDB_PG_FLAGS_BLOCKED;
        br_mdb_notify(brmctx->br->dev, mp, p, RTM_NEWMDB);

found:
        if (igmpv2_mldv1)
                mod_timer(&p->timer,
                          now + brmctx->multicast_membership_interval);

out:
        return p;
}

static int br_multicast_add_group(struct net_bridge_mcast *brmctx,
                                  struct net_bridge_mcast_port *pmctx,
                                  struct br_ip *group,
                                  const unsigned char *src,
                                  u8 filter_mode,
                                  bool igmpv2_mldv1)
{
        struct net_bridge_port_group *pg;
        int err;

        spin_lock(&brmctx->br->multicast_lock);
        pg = __br_multicast_add_group(brmctx, pmctx, group, src, filter_mode,
                                      igmpv2_mldv1, false);
        /* NULL is considered valid for host joined groups */
        err = PTR_ERR_OR_ZERO(pg);
        spin_unlock(&brmctx->br->multicast_lock);

        return err;
}

static int br_ip4_multicast_add_group(struct net_bridge_mcast *brmctx,
                                      struct net_bridge_mcast_port *pmctx,
                                      __be32 group,
                                      __u16 vid,
                                      const unsigned char *src,
                                      bool igmpv2)
{
        struct br_ip br_group;
        u8 filter_mode;

        if (ipv4_is_local_multicast(group))
                return 0;

        memset(&br_group, 0, sizeof(br_group));
        br_group.dst.ip4 = group;
        br_group.proto = htons(ETH_P_IP);
        br_group.vid = vid;
        filter_mode = igmpv2 ? MCAST_EXCLUDE : MCAST_INCLUDE;

        return br_multicast_add_group(brmctx, pmctx, &br_group, src,
                                      filter_mode, igmpv2);
}

#if IS_ENABLED(CONFIG_IPV6)
static int br_ip6_multicast_add_group(struct net_bridge_mcast *brmctx,
                                      struct net_bridge_mcast_port *pmctx,
                                      const struct in6_addr *group,
                                      __u16 vid,
                                      const unsigned char *src,
                                      bool mldv1)
{
        struct br_ip br_group;
        u8 filter_mode;

        if (ipv6_addr_is_ll_all_nodes(group))
                return 0;

        memset(&br_group, 0, sizeof(br_group));
        br_group.dst.ip6 = *group;
        br_group.proto = htons(ETH_P_IPV6);
        br_group.vid = vid;
        filter_mode = mldv1 ? MCAST_EXCLUDE : MCAST_INCLUDE;

        return br_multicast_add_group(brmctx, pmctx, &br_group, src,
                                      filter_mode, mldv1);
}
#endif

static bool br_multicast_rport_del(struct hlist_node *rlist)
{
        if (hlist_unhashed(rlist))
                return false;

        hlist_del_init_rcu(rlist);
        return true;
}

static bool br_ip4_multicast_rport_del(struct net_bridge_mcast_port *pmctx)
{
        return br_multicast_rport_del(&pmctx->ip4_rlist);
}

static bool br_ip6_multicast_rport_del(struct net_bridge_mcast_port *pmctx)
{
#if IS_ENABLED(CONFIG_IPV6)
        return br_multicast_rport_del(&pmctx->ip6_rlist);
#else
        return false;
#endif
}

static void br_multicast_router_expired(struct net_bridge_mcast_port *pmctx,
                                        struct timer_list *t,
                                        struct hlist_node *rlist)
{
        struct net_bridge *br = pmctx->port->br;
        bool del;

        spin_lock(&br->multicast_lock);
        if (pmctx->multicast_router == MDB_RTR_TYPE_DISABLED ||
            pmctx->multicast_router == MDB_RTR_TYPE_PERM ||
            timer_pending(t))
                goto out;

        del = br_multicast_rport_del(rlist);
        br_multicast_rport_del_notify(pmctx, del);
out:
        spin_unlock(&br->multicast_lock);
}

static void br_ip4_multicast_router_expired(struct timer_list *t)
{
        struct net_bridge_mcast_port *pmctx = from_timer(pmctx, t,
                                                         ip4_mc_router_timer);

        br_multicast_router_expired(pmctx, t, &pmctx->ip4_rlist);
}

#if IS_ENABLED(CONFIG_IPV6)
static void br_ip6_multicast_router_expired(struct timer_list *t)
{
        struct net_bridge_mcast_port *pmctx = from_timer(pmctx, t,
                                                         ip6_mc_router_timer);

        br_multicast_router_expired(pmctx, t, &pmctx->ip6_rlist);
}
#endif

static void br_mc_router_state_change(struct net_bridge *p,
                                      bool is_mc_router)
{
        struct switchdev_attr attr = {
                .orig_dev = p->dev,
                .id = SWITCHDEV_ATTR_ID_BRIDGE_MROUTER,
                .flags = SWITCHDEV_F_DEFER,
                .u.mrouter = is_mc_router,
        };

        switchdev_port_attr_set(p->dev, &attr, NULL);
}

static void br_multicast_local_router_expired(struct net_bridge_mcast *brmctx,
                                              struct timer_list *timer)
{
        spin_lock(&brmctx->br->multicast_lock);
        if (brmctx->multicast_router == MDB_RTR_TYPE_DISABLED ||
            brmctx->multicast_router == MDB_RTR_TYPE_PERM ||
            br_ip4_multicast_is_router(brmctx) ||
            br_ip6_multicast_is_router(brmctx))
                goto out;

        br_mc_router_state_change(brmctx->br, false);
out:
        spin_unlock(&brmctx->br->multicast_lock);
}

static void br_ip4_multicast_local_router_expired(struct timer_list *t)
{
        struct net_bridge_mcast *brmctx = from_timer(brmctx, t,
                                                     ip4_mc_router_timer);

        br_multicast_local_router_expired(brmctx, t);
}

#if IS_ENABLED(CONFIG_IPV6)
static void br_ip6_multicast_local_router_expired(struct timer_list *t)
{
        struct net_bridge_mcast *brmctx = from_timer(brmctx, t,
                                                     ip6_mc_router_timer);

        br_multicast_local_router_expired(brmctx, t);
}
#endif

static void br_multicast_querier_expired(struct net_bridge_mcast *brmctx,
                                         struct bridge_mcast_own_query *query)
{
        spin_lock(&brmctx->br->multicast_lock);
        if (!netif_running(brmctx->br->dev) ||
            br_multicast_ctx_vlan_global_disabled(brmctx) ||
            !br_opt_get(brmctx->br, BROPT_MULTICAST_ENABLED))
                goto out;

        br_multicast_start_querier(brmctx, query);

out:
        spin_unlock(&brmctx->br->multicast_lock);
}

static void br_ip4_multicast_querier_expired(struct timer_list *t)
{
        struct net_bridge_mcast *brmctx = from_timer(brmctx, t,
                                                     ip4_other_query.timer);

        br_multicast_querier_expired(brmctx, &brmctx->ip4_own_query);
}

#if IS_ENABLED(CONFIG_IPV6)
static void br_ip6_multicast_querier_expired(struct timer_list *t)
{
        struct net_bridge_mcast *brmctx = from_timer(brmctx, t,
                                                     ip6_other_query.timer);

        br_multicast_querier_expired(brmctx, &brmctx->ip6_own_query);
}
#endif

static void br_multicast_query_delay_expired(struct timer_list *t)
{
}

static void br_multicast_select_own_querier(struct net_bridge_mcast *brmctx,
                                            struct br_ip *ip,
                                            struct sk_buff *skb)
{
        if (ip->proto == htons(ETH_P_IP))
                brmctx->ip4_querier.addr.src.ip4 = ip_hdr(skb)->saddr;
#if IS_ENABLED(CONFIG_IPV6)
        else
                brmctx->ip6_querier.addr.src.ip6 = ipv6_hdr(skb)->saddr;
#endif
}

static void __br_multicast_send_query(struct net_bridge_mcast *brmctx,
                                      struct net_bridge_mcast_port *pmctx,
                                      struct net_bridge_port_group *pg,
                                      struct br_ip *ip_dst,
                                      struct br_ip *group,
                                      bool with_srcs,
                                      u8 sflag,
                                      bool *need_rexmit)
{
        bool over_lmqt = !!sflag;
        struct sk_buff *skb;
        u8 igmp_type;

        if (!br_multicast_ctx_should_use(brmctx, pmctx) ||
            !br_multicast_ctx_matches_vlan_snooping(brmctx))
                return;

again_under_lmqt:
        skb = br_multicast_alloc_query(brmctx, pmctx, pg, ip_dst, group,
                                       with_srcs, over_lmqt, sflag, &igmp_type,
                                       need_rexmit);
        if (!skb)
                return;

        if (pmctx) {
                skb->dev = pmctx->port->dev;
                br_multicast_count(brmctx->br, pmctx->port, skb, igmp_type,
                                   BR_MCAST_DIR_TX);
                NF_HOOK(NFPROTO_BRIDGE, NF_BR_LOCAL_OUT,
                        dev_net(pmctx->port->dev), NULL, skb, NULL, skb->dev,
                        br_dev_queue_push_xmit);

                if (over_lmqt && with_srcs && sflag) {
                        over_lmqt = false;
                        goto again_under_lmqt;
                }
        } else {
                br_multicast_select_own_querier(brmctx, group, skb);
                br_multicast_count(brmctx->br, NULL, skb, igmp_type,
                                   BR_MCAST_DIR_RX);
                netif_rx(skb);
        }
}

static void br_multicast_read_querier(const struct bridge_mcast_querier *querier,
                                      struct bridge_mcast_querier *dest)
{
        unsigned int seq;

        memset(dest, 0, sizeof(*dest));
        do {
                seq = read_seqcount_begin(&querier->seq);
                dest->port_ifidx = querier->port_ifidx;
                memcpy(&dest->addr, &querier->addr, sizeof(struct br_ip));
        } while (read_seqcount_retry(&querier->seq, seq));
}

static void br_multicast_update_querier(struct net_bridge_mcast *brmctx,
                                        struct bridge_mcast_querier *querier,
                                        int ifindex,
                                        struct br_ip *saddr)
{
        write_seqcount_begin(&querier->seq);
        querier->port_ifidx = ifindex;
        memcpy(&querier->addr, saddr, sizeof(*saddr));
        write_seqcount_end(&querier->seq);
}

static void br_multicast_send_query(struct net_bridge_mcast *brmctx,
                                    struct net_bridge_mcast_port *pmctx,
                                    struct bridge_mcast_own_query *own_query)
{
        struct bridge_mcast_other_query *other_query = NULL;
        struct bridge_mcast_querier *querier;
        struct br_ip br_group;
        unsigned long time;

        if (!br_multicast_ctx_should_use(brmctx, pmctx) ||
            !br_opt_get(brmctx->br, BROPT_MULTICAST_ENABLED) ||
            !brmctx->multicast_querier)
                return;

        memset(&br_group.dst, 0, sizeof(br_group.dst));

        if (pmctx ? (own_query == &pmctx->ip4_own_query) :
                    (own_query == &brmctx->ip4_own_query)) {
                querier = &brmctx->ip4_querier;
                other_query = &brmctx->ip4_other_query;
                br_group.proto = htons(ETH_P_IP);
#if IS_ENABLED(CONFIG_IPV6)
        } else {
                querier = &brmctx->ip6_querier;
                other_query = &brmctx->ip6_other_query;
                br_group.proto = htons(ETH_P_IPV6);
#endif
        }

        if (!other_query || timer_pending(&other_query->timer))
                return;

        /* we're about to select ourselves as querier */
        if (!pmctx && querier->port_ifidx) {
                struct br_ip zeroip = {};

                br_multicast_update_querier(brmctx, querier, 0, &zeroip);
        }

        __br_multicast_send_query(brmctx, pmctx, NULL, NULL, &br_group, false,
                                  0, NULL);

        time = jiffies;
        time += own_query->startup_sent < brmctx->multicast_startup_query_count ?
                brmctx->multicast_startup_query_interval :
                brmctx->multicast_query_interval;
        mod_timer(&own_query->timer, time);
}

static void
br_multicast_port_query_expired(struct net_bridge_mcast_port *pmctx,
                                struct bridge_mcast_own_query *query)
{
        struct net_bridge *br = pmctx->port->br;
        struct net_bridge_mcast *brmctx;

        spin_lock(&br->multicast_lock);
        if (br_multicast_port_ctx_state_stopped(pmctx))
                goto out;

        brmctx = br_multicast_port_ctx_get_global(pmctx);
        if (query->startup_sent < brmctx->multicast_startup_query_count)
                query->startup_sent++;

        br_multicast_send_query(brmctx, pmctx, query);

out:
        spin_unlock(&br->multicast_lock);
}

static void br_ip4_multicast_port_query_expired(struct timer_list *t)
{
        struct net_bridge_mcast_port *pmctx = from_timer(pmctx, t,
                                                         ip4_own_query.timer);

        br_multicast_port_query_expired(pmctx, &pmctx->ip4_own_query);
}

#if IS_ENABLED(CONFIG_IPV6)
static void br_ip6_multicast_port_query_expired(struct timer_list *t)
{
        struct net_bridge_mcast_port *pmctx = from_timer(pmctx, t,
                                                         ip6_own_query.timer);

        br_multicast_port_query_expired(pmctx, &pmctx->ip6_own_query);
}
#endif

static void br_multicast_port_group_rexmit(struct timer_list *t)
{
        struct net_bridge_port_group *pg = from_timer(pg, t, rexmit_timer);
        struct bridge_mcast_other_query *other_query = NULL;
        struct net_bridge *br = pg->key.port->br;
        struct net_bridge_mcast_port *pmctx;
        struct net_bridge_mcast *brmctx;
        bool need_rexmit = false;

        spin_lock(&br->multicast_lock);
        if (!netif_running(br->dev) || hlist_unhashed(&pg->mglist) ||
            !br_opt_get(br, BROPT_MULTICAST_ENABLED))
                goto out;

        pmctx = br_multicast_pg_to_port_ctx(pg);
        if (!pmctx)
                goto out;
        brmctx = br_multicast_port_ctx_get_global(pmctx);
        if (!brmctx->multicast_querier)
                goto out;

        if (pg->key.addr.proto == htons(ETH_P_IP))
                other_query = &brmctx->ip4_other_query;
#if IS_ENABLED(CONFIG_IPV6)
        else
                other_query = &brmctx->ip6_other_query;
#endif

        if (!other_query || timer_pending(&other_query->timer))
                goto out;

        if (pg->grp_query_rexmit_cnt) {
                pg->grp_query_rexmit_cnt--;
                __br_multicast_send_query(brmctx, pmctx, pg, &pg->key.addr,
                                          &pg->key.addr, false, 1, NULL);
        }
        __br_multicast_send_query(brmctx, pmctx, pg, &pg->key.addr,
                                  &pg->key.addr, true, 0, &need_rexmit);

        if (pg->grp_query_rexmit_cnt || need_rexmit)
                mod_timer(&pg->rexmit_timer, jiffies +
                                             brmctx->multicast_last_member_interval);
out:
        spin_unlock(&br->multicast_lock);
}

static int br_mc_disabled_update(struct net_device *dev, bool value,
                                 struct netlink_ext_ack *extack)
{
        struct switchdev_attr attr = {
                .orig_dev = dev,
                .id = SWITCHDEV_ATTR_ID_BRIDGE_MC_DISABLED,
                .flags = SWITCHDEV_F_DEFER,
                .u.mc_disabled = !value,
        };

        return switchdev_port_attr_set(dev, &attr, extack);
}

void br_multicast_port_ctx_init(struct net_bridge_port *port,
                                struct net_bridge_vlan *vlan,
                                struct net_bridge_mcast_port *pmctx)
{
        pmctx->port = port;
        pmctx->vlan = vlan;
        pmctx->multicast_router = MDB_RTR_TYPE_TEMP_QUERY;
        timer_setup(&pmctx->ip4_mc_router_timer,
                    br_ip4_multicast_router_expired, 0);
        timer_setup(&pmctx->ip4_own_query.timer,
                    br_ip4_multicast_port_query_expired, 0);
#if IS_ENABLED(CONFIG_IPV6)
        timer_setup(&pmctx->ip6_mc_router_timer,
                    br_ip6_multicast_router_expired, 0);
        timer_setup(&pmctx->ip6_own_query.timer,
                    br_ip6_multicast_port_query_expired, 0);
#endif
}

void br_multicast_port_ctx_deinit(struct net_bridge_mcast_port *pmctx)
{
#if IS_ENABLED(CONFIG_IPV6)
        del_timer_sync(&pmctx->ip6_mc_router_timer);
#endif
        del_timer_sync(&pmctx->ip4_mc_router_timer);
}

int br_multicast_add_port(struct net_bridge_port *port)
{
        int err;

        port->multicast_eht_hosts_limit = BR_MCAST_DEFAULT_EHT_HOSTS_LIMIT;
        br_multicast_port_ctx_init(port, NULL, &port->multicast_ctx);

        err = br_mc_disabled_update(port->dev,
                                    br_opt_get(port->br,
                                               BROPT_MULTICAST_ENABLED),
                                    NULL);
        if (err && err != -EOPNOTSUPP)
                return err;

        port->mcast_stats = netdev_alloc_pcpu_stats(struct bridge_mcast_stats);
        if (!port->mcast_stats)
                return -ENOMEM;

        return 0;
}

void br_multicast_del_port(struct net_bridge_port *port)
{
        struct net_bridge *br = port->br;
        struct net_bridge_port_group *pg;
        struct hlist_node *n;

        /* Take care of the remaining groups, only perm ones should be left */
        spin_lock_bh(&br->multicast_lock);
        hlist_for_each_entry_safe(pg, n, &port->mglist, mglist)
                br_multicast_find_del_pg(br, pg);
        spin_unlock_bh(&br->multicast_lock);
        flush_work(&br->mcast_gc_work);
        br_multicast_port_ctx_deinit(&port->multicast_ctx);
        free_percpu(port->mcast_stats);
}

static void br_multicast_enable(struct bridge_mcast_own_query *query)
{
        query->startup_sent = 0;

        if (try_to_del_timer_sync(&query->timer) >= 0 ||
            del_timer(&query->timer))
                mod_timer(&query->timer, jiffies);
}

static void __br_multicast_enable_port_ctx(struct net_bridge_mcast_port *pmctx)
{
        struct net_bridge *br = pmctx->port->br;
        struct net_bridge_mcast *brmctx;

        brmctx = br_multicast_port_ctx_get_global(pmctx);
        if (!br_opt_get(br, BROPT_MULTICAST_ENABLED) ||
            !netif_running(br->dev))
                return;

        br_multicast_enable(&pmctx->ip4_own_query);
#if IS_ENABLED(CONFIG_IPV6)
        br_multicast_enable(&pmctx->ip6_own_query);
#endif
        if (pmctx->multicast_router == MDB_RTR_TYPE_PERM) {
                br_ip4_multicast_add_router(brmctx, pmctx);
                br_ip6_multicast_add_router(brmctx, pmctx);
        }

        if (br_multicast_port_ctx_is_vlan(pmctx)) {
                struct net_bridge_port_group *pg;
                u32 n = 0;

                /* The mcast_n_groups counter might be wrong. First,
                 * BR_VLFLAG_MCAST_ENABLED is toggled before temporary entries
                 * are flushed, thus mcast_n_groups after the toggle does not
                 * reflect the true values. And second, permanent entries added
                 * while BR_VLFLAG_MCAST_ENABLED was disabled, are not reflected
                 * either. Thus we have to refresh the counter.
                 */

                hlist_for_each_entry(pg, &pmctx->port->mglist, mglist) {
                        if (pg->key.addr.vid == pmctx->vlan->vid)
                                n++;
                }
                WRITE_ONCE(pmctx->mdb_n_entries, n);
        }
}

void br_multicast_enable_port(struct net_bridge_port *port)
{
        struct net_bridge *br = port->br;

        spin_lock_bh(&br->multicast_lock);
        __br_multicast_enable_port_ctx(&port->multicast_ctx);
        spin_unlock_bh(&br->multicast_lock);
}

static void __br_multicast_disable_port_ctx(struct net_bridge_mcast_port *pmctx)
{
        struct net_bridge_port_group *pg;
        struct hlist_node *n;
        bool del = false;

        hlist_for_each_entry_safe(pg, n, &pmctx->port->mglist, mglist)
                if (!(pg->flags & MDB_PG_FLAGS_PERMANENT) &&
                    (!br_multicast_port_ctx_is_vlan(pmctx) ||
                     pg->key.addr.vid == pmctx->vlan->vid))
                        br_multicast_find_del_pg(pmctx->port->br, pg);

        del |= br_ip4_multicast_rport_del(pmctx);
        del_timer(&pmctx->ip4_mc_router_timer);
        del_timer(&pmctx->ip4_own_query.timer);
        del |= br_ip6_multicast_rport_del(pmctx);
#if IS_ENABLED(CONFIG_IPV6)
        del_timer(&pmctx->ip6_mc_router_timer);
        del_timer(&pmctx->ip6_own_query.timer);
#endif
        br_multicast_rport_del_notify(pmctx, del);
}

void br_multicast_disable_port(struct net_bridge_port *port)
{
        spin_lock_bh(&port->br->multicast_lock);
        __br_multicast_disable_port_ctx(&port->multicast_ctx);
        spin_unlock_bh(&port->br->multicast_lock);
}

static int __grp_src_delete_marked(struct net_bridge_port_group *pg)
{
        struct net_bridge_group_src *ent;
        struct hlist_node *tmp;
        int deleted = 0;

        hlist_for_each_entry_safe(ent, tmp, &pg->src_list, node)
                if (ent->flags & BR_SGRP_F_DELETE) {
                        br_multicast_del_group_src(ent, false);
                        deleted++;
                }

        return deleted;
}

static void __grp_src_mod_timer(struct net_bridge_group_src *src,
                                unsigned long expires)
{
        mod_timer(&src->timer, expires);
        br_multicast_fwd_src_handle(src);
}

static void __grp_src_query_marked_and_rexmit(struct net_bridge_mcast *brmctx,
                                              struct net_bridge_mcast_port *pmctx,
                                              struct net_bridge_port_group *pg)
{
        struct bridge_mcast_other_query *other_query = NULL;
        u32 lmqc = brmctx->multicast_last_member_count;
        unsigned long lmqt, lmi, now = jiffies;
        struct net_bridge_group_src *ent;

        if (!netif_running(brmctx->br->dev) ||
            !br_opt_get(brmctx->br, BROPT_MULTICAST_ENABLED))
                return;

        if (pg->key.addr.proto == htons(ETH_P_IP))
                other_query = &brmctx->ip4_other_query;
#if IS_ENABLED(CONFIG_IPV6)
        else
                other_query = &brmctx->ip6_other_query;
#endif

        lmqt = now + br_multicast_lmqt(brmctx);
        hlist_for_each_entry(ent, &pg->src_list, node) {
                if (ent->flags & BR_SGRP_F_SEND) {
                        ent->flags &= ~BR_SGRP_F_SEND;
                        if (ent->timer.expires > lmqt) {
                                if (brmctx->multicast_querier &&
                                    other_query &&
                                    !timer_pending(&other_query->timer))
                                        ent->src_query_rexmit_cnt = lmqc;
                                __grp_src_mod_timer(ent, lmqt);
                        }
                }
        }

        if (!brmctx->multicast_querier ||
            !other_query || timer_pending(&other_query->timer))
                return;

        __br_multicast_send_query(brmctx, pmctx, pg, &pg->key.addr,
                                  &pg->key.addr, true, 1, NULL);

        lmi = now + brmctx->multicast_last_member_interval;
        if (!timer_pending(&pg->rexmit_timer) ||
            time_after(pg->rexmit_timer.expires, lmi))
                mod_timer(&pg->rexmit_timer, lmi);
}

static void __grp_send_query_and_rexmit(struct net_bridge_mcast *brmctx,
                                        struct net_bridge_mcast_port *pmctx,
                                        struct net_bridge_port_group *pg)
{
        struct bridge_mcast_other_query *other_query = NULL;
        unsigned long now = jiffies, lmi;

        if (!netif_running(brmctx->br->dev) ||
            !br_opt_get(brmctx->br, BROPT_MULTICAST_ENABLED))
                return;

        if (pg->key.addr.proto == htons(ETH_P_IP))
                other_query = &brmctx->ip4_other_query;
#if IS_ENABLED(CONFIG_IPV6)
        else
                other_query = &brmctx->ip6_other_query;
#endif

        if (brmctx->multicast_querier &&
            other_query && !timer_pending(&other_query->timer)) {
                lmi = now + brmctx->multicast_last_member_interval;
                pg->grp_query_rexmit_cnt = brmctx->multicast_last_member_count - 1;
                __br_multicast_send_query(brmctx, pmctx, pg, &pg->key.addr,
                                          &pg->key.addr, false, 0, NULL);
                if (!timer_pending(&pg->rexmit_timer) ||
                    time_after(pg->rexmit_timer.expires, lmi))
                        mod_timer(&pg->rexmit_timer, lmi);
        }

        if (pg->filter_mode == MCAST_EXCLUDE &&
            (!timer_pending(&pg->timer) ||
             time_after(pg->timer.expires, now + br_multicast_lmqt(brmctx))))
                mod_timer(&pg->timer, now + br_multicast_lmqt(brmctx));
}

/* State          Msg type      New state                Actions
 * INCLUDE (A)    IS_IN (B)     INCLUDE (A+B)            (B)=GMI
 * INCLUDE (A)    ALLOW (B)     INCLUDE (A+B)            (B)=GMI
 * EXCLUDE (X,Y)  ALLOW (A)     EXCLUDE (X+A,Y-A)        (A)=GMI
 */
static bool br_multicast_isinc_allow(const struct net_bridge_mcast *brmctx,
                                     struct net_bridge_port_group *pg, void *h_addr,
                                     void *srcs, u32 nsrcs, size_t addr_size,
                                     int grec_type)
{
        struct net_bridge_group_src *ent;
        unsigned long now = jiffies;
        bool changed = false;
        struct br_ip src_ip;
        u32 src_idx;

        memset(&src_ip, 0, sizeof(src_ip));
        src_ip.proto = pg->key.addr.proto;
        for (src_idx = 0; src_idx < nsrcs; src_idx++) {
                memcpy(&src_ip.src, srcs + (src_idx * addr_size), addr_size);
                ent = br_multicast_find_group_src(pg, &src_ip);
                if (!ent) {
                        ent = br_multicast_new_group_src(pg, &src_ip);
                        if (ent)
                                changed = true;
                }

                if (ent)
                        __grp_src_mod_timer(ent, now + br_multicast_gmi(brmctx));
        }

        if (br_multicast_eht_handle(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                    grec_type))
                changed = true;

        return changed;
}

/* State          Msg type      New state                Actions
 * INCLUDE (A)    IS_EX (B)     EXCLUDE (A*B,B-A)        (B-A)=0
 *                                                       Delete (A-B)
 *                                                       Group Timer=GMI
 */
static void __grp_src_isexc_incl(const struct net_bridge_mcast *brmctx,
                                 struct net_bridge_port_group *pg, void *h_addr,
                                 void *srcs, u32 nsrcs, size_t addr_size,
                                 int grec_type)
{
        struct net_bridge_group_src *ent;
        struct br_ip src_ip;
        u32 src_idx;

        hlist_for_each_entry(ent, &pg->src_list, node)
                ent->flags |= BR_SGRP_F_DELETE;

        memset(&src_ip, 0, sizeof(src_ip));
        src_ip.proto = pg->key.addr.proto;
        for (src_idx = 0; src_idx < nsrcs; src_idx++) {
                memcpy(&src_ip.src, srcs + (src_idx * addr_size), addr_size);
                ent = br_multicast_find_group_src(pg, &src_ip);
                if (ent)
                        ent->flags &= ~BR_SGRP_F_DELETE;
                else
                        ent = br_multicast_new_group_src(pg, &src_ip);
                if (ent)
                        br_multicast_fwd_src_handle(ent);
        }

        br_multicast_eht_handle(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                grec_type);

        __grp_src_delete_marked(pg);
}

/* State          Msg type      New state                Actions
 * EXCLUDE (X,Y)  IS_EX (A)     EXCLUDE (A-Y,Y*A)        (A-X-Y)=GMI
 *                                                       Delete (X-A)
 *                                                       Delete (Y-A)
 *                                                       Group Timer=GMI
 */
static bool __grp_src_isexc_excl(const struct net_bridge_mcast *brmctx,
                                 struct net_bridge_port_group *pg, void *h_addr,
                                 void *srcs, u32 nsrcs, size_t addr_size,
                                 int grec_type)
{
        struct net_bridge_group_src *ent;
        unsigned long now = jiffies;
        bool changed = false;
        struct br_ip src_ip;
        u32 src_idx;

        hlist_for_each_entry(ent, &pg->src_list, node)
                ent->flags |= BR_SGRP_F_DELETE;

        memset(&src_ip, 0, sizeof(src_ip));
        src_ip.proto = pg->key.addr.proto;
        for (src_idx = 0; src_idx < nsrcs; src_idx++) {
                memcpy(&src_ip.src, srcs + (src_idx * addr_size), addr_size);
                ent = br_multicast_find_group_src(pg, &src_ip);
                if (ent) {
                        ent->flags &= ~BR_SGRP_F_DELETE;
                } else {
                        ent = br_multicast_new_group_src(pg, &src_ip);
                        if (ent) {
                                __grp_src_mod_timer(ent,
                                                    now + br_multicast_gmi(brmctx));
                                changed = true;
                        }
                }
        }

        if (br_multicast_eht_handle(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                    grec_type))
                changed = true;

        if (__grp_src_delete_marked(pg))
                changed = true;

        return changed;
}

static bool br_multicast_isexc(const struct net_bridge_mcast *brmctx,
                               struct net_bridge_port_group *pg, void *h_addr,
                               void *srcs, u32 nsrcs, size_t addr_size,
                               int grec_type)
{
        bool changed = false;

        switch (pg->filter_mode) {
        case MCAST_INCLUDE:
                __grp_src_isexc_incl(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                     grec_type);
                br_multicast_star_g_handle_mode(pg, MCAST_EXCLUDE);
                changed = true;
                break;
        case MCAST_EXCLUDE:
                changed = __grp_src_isexc_excl(brmctx, pg, h_addr, srcs, nsrcs,
                                               addr_size, grec_type);
                break;
        }

        pg->filter_mode = MCAST_EXCLUDE;
        mod_timer(&pg->timer, jiffies + br_multicast_gmi(brmctx));

        return changed;
}

/* State          Msg type      New state                Actions
 * INCLUDE (A)    TO_IN (B)     INCLUDE (A+B)            (B)=GMI
 *                                                       Send Q(G,A-B)
 */
static bool __grp_src_toin_incl(struct net_bridge_mcast *brmctx,
                                struct net_bridge_mcast_port *pmctx,
                                struct net_bridge_port_group *pg, void *h_addr,
                                void *srcs, u32 nsrcs, size_t addr_size,
                                int grec_type)
{
        u32 src_idx, to_send = pg->src_ents;
        struct net_bridge_group_src *ent;
        unsigned long now = jiffies;
        bool changed = false;
        struct br_ip src_ip;

        hlist_for_each_entry(ent, &pg->src_list, node)
                ent->flags |= BR_SGRP_F_SEND;

        memset(&src_ip, 0, sizeof(src_ip));
        src_ip.proto = pg->key.addr.proto;
        for (src_idx = 0; src_idx < nsrcs; src_idx++) {
                memcpy(&src_ip.src, srcs + (src_idx * addr_size), addr_size);
                ent = br_multicast_find_group_src(pg, &src_ip);
                if (ent) {
                        ent->flags &= ~BR_SGRP_F_SEND;
                        to_send--;
                } else {
                        ent = br_multicast_new_group_src(pg, &src_ip);
                        if (ent)
                                changed = true;
                }
                if (ent)
                        __grp_src_mod_timer(ent, now + br_multicast_gmi(brmctx));
        }

        if (br_multicast_eht_handle(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                    grec_type))
                changed = true;

        if (to_send)
                __grp_src_query_marked_and_rexmit(brmctx, pmctx, pg);

        return changed;
}

/* State          Msg type      New state                Actions
 * EXCLUDE (X,Y)  TO_IN (A)     EXCLUDE (X+A,Y-A)        (A)=GMI
 *                                                       Send Q(G,X-A)
 *                                                       Send Q(G)
 */
static bool __grp_src_toin_excl(struct net_bridge_mcast *brmctx,
                                struct net_bridge_mcast_port *pmctx,
                                struct net_bridge_port_group *pg, void *h_addr,
                                void *srcs, u32 nsrcs, size_t addr_size,
                                int grec_type)
{
        u32 src_idx, to_send = pg->src_ents;
        struct net_bridge_group_src *ent;
        unsigned long now = jiffies;
        bool changed = false;
        struct br_ip src_ip;

        hlist_for_each_entry(ent, &pg->src_list, node)
                if (timer_pending(&ent->timer))
                        ent->flags |= BR_SGRP_F_SEND;

        memset(&src_ip, 0, sizeof(src_ip));
        src_ip.proto = pg->key.addr.proto;
        for (src_idx = 0; src_idx < nsrcs; src_idx++) {
                memcpy(&src_ip.src, srcs + (src_idx * addr_size), addr_size);
                ent = br_multicast_find_group_src(pg, &src_ip);
                if (ent) {
                        if (timer_pending(&ent->timer)) {
                                ent->flags &= ~BR_SGRP_F_SEND;
                                to_send--;
                        }
                } else {
                        ent = br_multicast_new_group_src(pg, &src_ip);
                        if (ent)
                                changed = true;
                }
                if (ent)
                        __grp_src_mod_timer(ent, now + br_multicast_gmi(brmctx));
        }

        if (br_multicast_eht_handle(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                    grec_type))
                changed = true;

        if (to_send)
                __grp_src_query_marked_and_rexmit(brmctx, pmctx, pg);

        __grp_send_query_and_rexmit(brmctx, pmctx, pg);

        return changed;
}

static bool br_multicast_toin(struct net_bridge_mcast *brmctx,
                              struct net_bridge_mcast_port *pmctx,
                              struct net_bridge_port_group *pg, void *h_addr,
                              void *srcs, u32 nsrcs, size_t addr_size,
                              int grec_type)
{
        bool changed = false;

        switch (pg->filter_mode) {
        case MCAST_INCLUDE:
                changed = __grp_src_toin_incl(brmctx, pmctx, pg, h_addr, srcs,
                                              nsrcs, addr_size, grec_type);
                break;
        case MCAST_EXCLUDE:
                changed = __grp_src_toin_excl(brmctx, pmctx, pg, h_addr, srcs,
                                              nsrcs, addr_size, grec_type);
                break;
        }

        if (br_multicast_eht_should_del_pg(pg)) {
                pg->flags |= MDB_PG_FLAGS_FAST_LEAVE;
                br_multicast_find_del_pg(pg->key.port->br, pg);
                /* a notification has already been sent and we shouldn't
                 * access pg after the delete so we have to return false
                 */
                changed = false;
        }

        return changed;
}

/* State          Msg type      New state                Actions
 * INCLUDE (A)    TO_EX (B)     EXCLUDE (A*B,B-A)        (B-A)=0
 *                                                       Delete (A-B)
 *                                                       Send Q(G,A*B)
 *                                                       Group Timer=GMI
 */
static void __grp_src_toex_incl(struct net_bridge_mcast *brmctx,
                                struct net_bridge_mcast_port *pmctx,
                                struct net_bridge_port_group *pg, void *h_addr,
                                void *srcs, u32 nsrcs, size_t addr_size,
                                int grec_type)
{
        struct net_bridge_group_src *ent;
        u32 src_idx, to_send = 0;
        struct br_ip src_ip;

        hlist_for_each_entry(ent, &pg->src_list, node)
                ent->flags = (ent->flags & ~BR_SGRP_F_SEND) | BR_SGRP_F_DELETE;

        memset(&src_ip, 0, sizeof(src_ip));
        src_ip.proto = pg->key.addr.proto;
        for (src_idx = 0; src_idx < nsrcs; src_idx++) {
                memcpy(&src_ip.src, srcs + (src_idx * addr_size), addr_size);
                ent = br_multicast_find_group_src(pg, &src_ip);
                if (ent) {
                        ent->flags = (ent->flags & ~BR_SGRP_F_DELETE) |
                                     BR_SGRP_F_SEND;
                        to_send++;
                } else {
                        ent = br_multicast_new_group_src(pg, &src_ip);
                }
                if (ent)
                        br_multicast_fwd_src_handle(ent);
        }

        br_multicast_eht_handle(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                grec_type);

        __grp_src_delete_marked(pg);
        if (to_send)
                __grp_src_query_marked_and_rexmit(brmctx, pmctx, pg);
}

/* State          Msg type      New state                Actions
 * EXCLUDE (X,Y)  TO_EX (A)     EXCLUDE (A-Y,Y*A)        (A-X-Y)=Group Timer
 *                                                       Delete (X-A)
 *                                                       Delete (Y-A)
 *                                                       Send Q(G,A-Y)
 *                                                       Group Timer=GMI
 */
static bool __grp_src_toex_excl(struct net_bridge_mcast *brmctx,
                                struct net_bridge_mcast_port *pmctx,
                                struct net_bridge_port_group *pg, void *h_addr,
                                void *srcs, u32 nsrcs, size_t addr_size,
                                int grec_type)
{
        struct net_bridge_group_src *ent;
        u32 src_idx, to_send = 0;
        bool changed = false;
        struct br_ip src_ip;

        hlist_for_each_entry(ent, &pg->src_list, node)
                ent->flags = (ent->flags & ~BR_SGRP_F_SEND) | BR_SGRP_F_DELETE;

        memset(&src_ip, 0, sizeof(src_ip));
        src_ip.proto = pg->key.addr.proto;
        for (src_idx = 0; src_idx < nsrcs; src_idx++) {
                memcpy(&src_ip.src, srcs + (src_idx * addr_size), addr_size);
                ent = br_multicast_find_group_src(pg, &src_ip);
                if (ent) {
                        ent->flags &= ~BR_SGRP_F_DELETE;
                } else {
                        ent = br_multicast_new_group_src(pg, &src_ip);
                        if (ent) {
                                __grp_src_mod_timer(ent, pg->timer.expires);
                                changed = true;
                        }
                }
                if (ent && timer_pending(&ent->timer)) {
                        ent->flags |= BR_SGRP_F_SEND;
                        to_send++;
                }
        }

        if (br_multicast_eht_handle(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                    grec_type))
                changed = true;

        if (__grp_src_delete_marked(pg))
                changed = true;
        if (to_send)
                __grp_src_query_marked_and_rexmit(brmctx, pmctx, pg);

        return changed;
}

static bool br_multicast_toex(struct net_bridge_mcast *brmctx,
                              struct net_bridge_mcast_port *pmctx,
                              struct net_bridge_port_group *pg, void *h_addr,
                              void *srcs, u32 nsrcs, size_t addr_size,
                              int grec_type)
{
        bool changed = false;

        switch (pg->filter_mode) {
        case MCAST_INCLUDE:
                __grp_src_toex_incl(brmctx, pmctx, pg, h_addr, srcs, nsrcs,
                                    addr_size, grec_type);
                br_multicast_star_g_handle_mode(pg, MCAST_EXCLUDE);
                changed = true;
                break;
        case MCAST_EXCLUDE:
                changed = __grp_src_toex_excl(brmctx, pmctx, pg, h_addr, srcs,
                                              nsrcs, addr_size, grec_type);
                break;
        }

        pg->filter_mode = MCAST_EXCLUDE;
        mod_timer(&pg->timer, jiffies + br_multicast_gmi(brmctx));

        return changed;
}

/* State          Msg type      New state                Actions
 * INCLUDE (A)    BLOCK (B)     INCLUDE (A)              Send Q(G,A*B)
 */
static bool __grp_src_block_incl(struct net_bridge_mcast *brmctx,
                                 struct net_bridge_mcast_port *pmctx,
                                 struct net_bridge_port_group *pg, void *h_addr,
                                 void *srcs, u32 nsrcs, size_t addr_size, int grec_type)
{
        struct net_bridge_group_src *ent;
        u32 src_idx, to_send = 0;
        bool changed = false;
        struct br_ip src_ip;

        hlist_for_each_entry(ent, &pg->src_list, node)
                ent->flags &= ~BR_SGRP_F_SEND;

        memset(&src_ip, 0, sizeof(src_ip));
        src_ip.proto = pg->key.addr.proto;
        for (src_idx = 0; src_idx < nsrcs; src_idx++) {
                memcpy(&src_ip.src, srcs + (src_idx * addr_size), addr_size);
                ent = br_multicast_find_group_src(pg, &src_ip);
                if (ent) {
                        ent->flags |= BR_SGRP_F_SEND;
                        to_send++;
                }
        }

        if (br_multicast_eht_handle(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                    grec_type))
                changed = true;

        if (to_send)
                __grp_src_query_marked_and_rexmit(brmctx, pmctx, pg);

        return changed;
}

/* State          Msg type      New state                Actions
 * EXCLUDE (X,Y)  BLOCK (A)     EXCLUDE (X+(A-Y),Y)      (A-X-Y)=Group Timer
 *                                                       Send Q(G,A-Y)
 */
static bool __grp_src_block_excl(struct net_bridge_mcast *brmctx,
                                 struct net_bridge_mcast_port *pmctx,
                                 struct net_bridge_port_group *pg, void *h_addr,
                                 void *srcs, u32 nsrcs, size_t addr_size, int grec_type)
{
        struct net_bridge_group_src *ent;
        u32 src_idx, to_send = 0;
        bool changed = false;
        struct br_ip src_ip;

        hlist_for_each_entry(ent, &pg->src_list, node)
                ent->flags &= ~BR_SGRP_F_SEND;

        memset(&src_ip, 0, sizeof(src_ip));
        src_ip.proto = pg->key.addr.proto;
        for (src_idx = 0; src_idx < nsrcs; src_idx++) {
                memcpy(&src_ip.src, srcs + (src_idx * addr_size), addr_size);
                ent = br_multicast_find_group_src(pg, &src_ip);
                if (!ent) {
                        ent = br_multicast_new_group_src(pg, &src_ip);
                        if (ent) {
                                __grp_src_mod_timer(ent, pg->timer.expires);
                                changed = true;
                        }
                }
                if (ent && timer_pending(&ent->timer)) {
                        ent->flags |= BR_SGRP_F_SEND;
                        to_send++;
                }
        }

        if (br_multicast_eht_handle(brmctx, pg, h_addr, srcs, nsrcs, addr_size,
                                    grec_type))
                changed = true;

        if (to_send)
                __grp_src_query_marked_and_rexmit(brmctx, pmctx, pg);

        return changed;
}

static bool br_multicast_block(struct net_bridge_mcast *brmctx,
                               struct net_bridge_mcast_port *pmctx,
                               struct net_bridge_port_group *pg, void *h_addr,
                               void *srcs, u32 nsrcs, size_t addr_size, int grec_type)
{
        bool changed = false;

        switch (pg->filter_mode) {
        case MCAST_INCLUDE:
                changed = __grp_src_block_incl(brmctx, pmctx, pg, h_addr, srcs,
                                               nsrcs, addr_size, grec_type);
                break;
        case MCAST_EXCLUDE:
                changed = __grp_src_block_excl(brmctx, pmctx, pg, h_addr, srcs,
                                               nsrcs, addr_size, grec_type);
                break;
        }

        if ((pg->filter_mode == MCAST_INCLUDE && hlist_empty(&pg->src_list)) ||
            br_multicast_eht_should_del_pg(pg)) {
                if (br_multicast_eht_should_del_pg(pg))
                        pg->flags |= MDB_PG_FLAGS_FAST_LEAVE;
                br_multicast_find_del_pg(pg->key.port->br, pg);
                /* a notification has already been sent and we shouldn't
                 * access pg after the delete so we have to return false
                 */
                changed = false;
        }

        return changed;
}

static struct net_bridge_port_group *
br_multicast_find_port(struct net_bridge_mdb_entry *mp,
                       struct net_bridge_port *p,
                       const unsigned char *src)
{
        struct net_bridge *br __maybe_unused = mp->br;
        struct net_bridge_port_group *pg;

        for (pg = mlock_dereference(mp->ports, br);
             pg;
             pg = mlock_dereference(pg->next, br))
                if (br_port_group_equal(pg, p, src))
                        return pg;

        return NULL;
}

static int br_ip4_multicast_igmp3_report(struct net_bridge_mcast *brmctx,
                                         struct net_bridge_mcast_port *pmctx,
                                         struct sk_buff *skb,
                                         u16 vid)
{
        bool igmpv2 = brmctx->multicast_igmp_version == 2;
        struct net_bridge_mdb_entry *mdst;
        struct net_bridge_port_group *pg;
        const unsigned char *src;
        struct igmpv3_report *ih;
        struct igmpv3_grec *grec;
        int i, len, num, type;
        __be32 group, *h_addr;
        bool changed = false;
        int err = 0;
        u16 nsrcs;

        ih = igmpv3_report_hdr(skb);
        num = ntohs(ih->ngrec);
        len = skb_transport_offset(skb) + sizeof(*ih);

        for (i = 0; i < num; i++) {
                len += sizeof(*grec);
                if (!ip_mc_may_pull(skb, len))
                        return -EINVAL;

                grec = (void *)(skb->data + len - sizeof(*grec));
                group = grec->grec_mca;
                type = grec->grec_type;
                nsrcs = ntohs(grec->grec_nsrcs);

                len += nsrcs * 4;
                if (!ip_mc_may_pull(skb, len))
                        return -EINVAL;

                switch (type) {
                case IGMPV3_MODE_IS_INCLUDE:
                case IGMPV3_MODE_IS_EXCLUDE:
                case IGMPV3_CHANGE_TO_INCLUDE:
                case IGMPV3_CHANGE_TO_EXCLUDE:
                case IGMPV3_ALLOW_NEW_SOURCES:
                case IGMPV3_BLOCK_OLD_SOURCES:
                        break;

                default:
                        continue;
                }

                src = eth_hdr(skb)->h_source;
                if (nsrcs == 0 &&
                    (type == IGMPV3_CHANGE_TO_INCLUDE ||
                     type == IGMPV3_MODE_IS_INCLUDE)) {
                        if (!pmctx || igmpv2) {
                                br_ip4_multicast_leave_group(brmctx, pmctx,
                                                             group, vid, src);
                                continue;
                        }
                } else {
                        err = br_ip4_multicast_add_group(brmctx, pmctx, group,
                                                         vid, src, igmpv2);
                        if (err)
                                break;
                }

                if (!pmctx || igmpv2)
                        continue;

                spin_lock(&brmctx->br->multicast_lock);
                if (!br_multicast_ctx_should_use(brmctx, pmctx))
                        goto unlock_continue;

                mdst = br_mdb_ip4_get(brmctx->br, group, vid);
                if (!mdst)
                        goto unlock_continue;
                pg = br_multicast_find_port(mdst, pmctx->port, src);
                if (!pg || (pg->flags & MDB_PG_FLAGS_PERMANENT))
                        goto unlock_continue;
                /* reload grec and host addr */
                grec = (void *)(skb->data + len - sizeof(*grec) - (nsrcs * 4));
                h_addr = &ip_hdr(skb)->saddr;
                switch (type) {
                case IGMPV3_ALLOW_NEW_SOURCES:
                        changed = br_multicast_isinc_allow(brmctx, pg, h_addr,
                                                           grec->grec_src,
                                                           nsrcs, sizeof(__be32), type);
                        break;
                case IGMPV3_MODE_IS_INCLUDE:
                        changed = br_multicast_isinc_allow(brmctx, pg, h_addr,
                                                           grec->grec_src,
                                                           nsrcs, sizeof(__be32), type);
                        break;
                case IGMPV3_MODE_IS_EXCLUDE:
                        changed = br_multicast_isexc(brmctx, pg, h_addr,
                                                     grec->grec_src,
                                                     nsrcs, sizeof(__be32), type);
                        break;
                case IGMPV3_CHANGE_TO_INCLUDE:
                        changed = br_multicast_toin(brmctx, pmctx, pg, h_addr,
                                                    grec->grec_src,
                                                    nsrcs, sizeof(__be32), type);
                        break;
                case IGMPV3_CHANGE_TO_EXCLUDE:
                        changed = br_multicast_toex(brmctx, pmctx, pg, h_addr,
                                                    grec->grec_src,
                                                    nsrcs, sizeof(__be32), type);
                        break;
                case IGMPV3_BLOCK_OLD_SOURCES:
                        changed = br_multicast_block(brmctx, pmctx, pg, h_addr,
                                                     grec->grec_src,
                                                     nsrcs, sizeof(__be32), type);
                        break;
                }
                if (changed)
                        br_mdb_notify(brmctx->br->dev, mdst, pg, RTM_NEWMDB);
unlock_continue:
                spin_unlock(&brmctx->br->multicast_lock);
        }

        return err;
}

#if IS_ENABLED(CONFIG_IPV6)
static int br_ip6_multicast_mld2_report(struct net_bridge_mcast *brmctx,
                                        struct net_bridge_mcast_port *pmctx,
                                        struct sk_buff *skb,
                                        u16 vid)
{
        bool mldv1 = brmctx->multicast_mld_version == 1;
        struct net_bridge_mdb_entry *mdst;
        struct net_bridge_port_group *pg;
        unsigned int nsrcs_offset;
        struct mld2_report *mld2r;
        const unsigned char *src;
        struct in6_addr *h_addr;
        struct mld2_grec *grec;
        unsigned int grec_len;
        bool changed = false;
        int i, len, num;
        int err = 0;

        if (!ipv6_mc_may_pull(skb, sizeof(*mld2r)))
                return -EINVAL;

        mld2r = (struct mld2_report *)icmp6_hdr(skb);
        num = ntohs(mld2r->mld2r_ngrec);
        len = skb_transport_offset(skb) + sizeof(*mld2r);

        for (i = 0; i < num; i++) {
                __be16 *_nsrcs, __nsrcs;
                u16 nsrcs;

                nsrcs_offset = len + offsetof(struct mld2_grec, grec_nsrcs);

                if (skb_transport_offset(skb) + ipv6_transport_len(skb) <
                    nsrcs_offset + sizeof(__nsrcs))
                        return -EINVAL;

                _nsrcs = skb_header_pointer(skb, nsrcs_offset,
                                            sizeof(__nsrcs), &__nsrcs);
                if (!_nsrcs)
                        return -EINVAL;

                nsrcs = ntohs(*_nsrcs);
                grec_len = struct_size(grec, grec_src, nsrcs);

                if (!ipv6_mc_may_pull(skb, len + grec_len))
                        return -EINVAL;

                grec = (struct mld2_grec *)(skb->data + len);
                len += grec_len;

                switch (grec->grec_type) {
                case MLD2_MODE_IS_INCLUDE:
                case MLD2_MODE_IS_EXCLUDE:
                case MLD2_CHANGE_TO_INCLUDE:
                case MLD2_CHANGE_TO_EXCLUDE:
                case MLD2_ALLOW_NEW_SOURCES:
                case MLD2_BLOCK_OLD_SOURCES:
                        break;

                default:
                        continue;
                }

                src = eth_hdr(skb)->h_source;
                if ((grec->grec_type == MLD2_CHANGE_TO_INCLUDE ||
                     grec->grec_type == MLD2_MODE_IS_INCLUDE) &&
                    nsrcs == 0) {
                        if (!pmctx || mldv1) {
                                br_ip6_multicast_leave_group(brmctx, pmctx,
                                                             &grec->grec_mca,
                                                             vid, src);
                                continue;
                        }
                } else {
                        err = br_ip6_multicast_add_group(brmctx, pmctx,
                                                         &grec->grec_mca, vid,
                                                         src, mldv1);
                        if (err)
                                break;
                }

                if (!pmctx || mldv1)
                        continue;

                spin_lock(&brmctx->br->multicast_lock);
                if (!br_multicast_ctx_should_use(brmctx, pmctx))
                        goto unlock_continue;

                mdst = br_mdb_ip6_get(brmctx->br, &grec->grec_mca, vid);
                if (!mdst)
                        goto unlock_continue;
                pg = br_multicast_find_port(mdst, pmctx->port, src);
                if (!pg || (pg->flags & MDB_PG_FLAGS_PERMANENT))
                        goto unlock_continue;
                h_addr = &ipv6_hdr(skb)->saddr;
                switch (grec->grec_type) {
                case MLD2_ALLOW_NEW_SOURCES:
                        changed = br_multicast_isinc_allow(brmctx, pg, h_addr,
                                                           grec->grec_src, nsrcs,
                                                           sizeof(struct in6_addr),
                                                           grec->grec_type);
                        break;
                case MLD2_MODE_IS_INCLUDE:
                        changed = br_multicast_isinc_allow(brmctx, pg, h_addr,
                                                           grec->grec_src, nsrcs,
                                                           sizeof(struct in6_addr),
                                                           grec->grec_type);
                        break;
                case MLD2_MODE_IS_EXCLUDE:
                        changed = br_multicast_isexc(brmctx, pg, h_addr,
                                                     grec->grec_src, nsrcs,
                                                     sizeof(struct in6_addr),
                                                     grec->grec_type);
                        break;
                case MLD2_CHANGE_TO_INCLUDE:
                        changed = br_multicast_toin(brmctx, pmctx, pg, h_addr,
                                                    grec->grec_src, nsrcs,
                                                    sizeof(struct in6_addr),
                                                    grec->grec_type);
                        break;
                case MLD2_CHANGE_TO_EXCLUDE:
                        changed = br_multicast_toex(brmctx, pmctx, pg, h_addr,
                                                    grec->grec_src, nsrcs,
                                                    sizeof(struct in6_addr),
                                                    grec->grec_type);
                        break;
                case MLD2_BLOCK_OLD_SOURCES:
                        changed = br_multicast_block(brmctx, pmctx, pg, h_addr,
                                                     grec->grec_src, nsrcs,
                                                     sizeof(struct in6_addr),
                                                     grec->grec_type);
                        break;
                }
                if (changed)
                        br_mdb_notify(brmctx->br->dev, mdst, pg, RTM_NEWMDB);
unlock_continue:
                spin_unlock(&brmctx->br->multicast_lock);
        }

        return err;
}
#endif

static bool br_multicast_select_querier(struct net_bridge_mcast *brmctx,
                                        struct net_bridge_mcast_port *pmctx,
                                        struct br_ip *saddr)
{
        int port_ifidx = pmctx ? pmctx->port->dev->ifindex : 0;
        struct timer_list *own_timer, *other_timer;
        struct bridge_mcast_querier *querier;

        switch (saddr->proto) {
        case htons(ETH_P_IP):
                querier = &brmctx->ip4_querier;
                own_timer = &brmctx->ip4_own_query.timer;
                other_timer = &brmctx->ip4_other_query.timer;
                if (!querier->addr.src.ip4 ||
                    ntohl(saddr->src.ip4) <= ntohl(querier->addr.src.ip4))
                        goto update;
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                querier = &brmctx->ip6_querier;
                own_timer = &brmctx->ip6_own_query.timer;
                other_timer = &brmctx->ip6_other_query.timer;
                if (ipv6_addr_cmp(&saddr->src.ip6, &querier->addr.src.ip6) <= 0)
                        goto update;
                break;
#endif
        default:
                return false;
        }

        if (!timer_pending(own_timer) && !timer_pending(other_timer))
                goto update;

        return false;

update:
        br_multicast_update_querier(brmctx, querier, port_ifidx, saddr);

        return true;
}

static struct net_bridge_port *
__br_multicast_get_querier_port(struct net_bridge *br,
                                const struct bridge_mcast_querier *querier)
{
        int port_ifidx = READ_ONCE(querier->port_ifidx);
        struct net_bridge_port *p;
        struct net_device *dev;

        if (port_ifidx == 0)
                return NULL;

        dev = dev_get_by_index_rcu(dev_net(br->dev), port_ifidx);
        if (!dev)
                return NULL;
        p = br_port_get_rtnl_rcu(dev);
        if (!p || p->br != br)
                return NULL;

        return p;
}

size_t br_multicast_querier_state_size(void)
{
        return nla_total_size(0) +                /* nest attribute */
               nla_total_size(sizeof(__be32)) + /* BRIDGE_QUERIER_IP_ADDRESS */
               nla_total_size(sizeof(int)) +    /* BRIDGE_QUERIER_IP_PORT */
               nla_total_size_64bit(sizeof(u64)) + /* BRIDGE_QUERIER_IP_OTHER_TIMER */
#if IS_ENABLED(CONFIG_IPV6)
               nla_total_size(sizeof(struct in6_addr)) + /* BRIDGE_QUERIER_IPV6_ADDRESS */
               nla_total_size(sizeof(int)) +                 /* BRIDGE_QUERIER_IPV6_PORT */
               nla_total_size_64bit(sizeof(u64)) +         /* BRIDGE_QUERIER_IPV6_OTHER_TIMER */
#endif
               0;
}

/* protected by rtnl or rcu */
int br_multicast_dump_querier_state(struct sk_buff *skb,
                                    const struct net_bridge_mcast *brmctx,
                                    int nest_attr)
{
        struct bridge_mcast_querier querier = {};
        struct net_bridge_port *p;
        struct nlattr *nest;

        if (!br_opt_get(brmctx->br, BROPT_MULTICAST_ENABLED) ||
            br_multicast_ctx_vlan_global_disabled(brmctx))
                return 0;

        nest = nla_nest_start(skb, nest_attr);
        if (!nest)
                return -EMSGSIZE;

        rcu_read_lock();
        if (!brmctx->multicast_querier &&
            !timer_pending(&brmctx->ip4_other_query.timer))
                goto out_v6;

        br_multicast_read_querier(&brmctx->ip4_querier, &querier);
        if (nla_put_in_addr(skb, BRIDGE_QUERIER_IP_ADDRESS,
                            querier.addr.src.ip4)) {
                rcu_read_unlock();
                goto out_err;
        }

        p = __br_multicast_get_querier_port(brmctx->br, &querier);
        if (timer_pending(&brmctx->ip4_other_query.timer) &&
            (nla_put_u64_64bit(skb, BRIDGE_QUERIER_IP_OTHER_TIMER,
                               br_timer_value(&brmctx->ip4_other_query.timer),
                               BRIDGE_QUERIER_PAD) ||
             (p && nla_put_u32(skb, BRIDGE_QUERIER_IP_PORT, p->dev->ifindex)))) {
                rcu_read_unlock();
                goto out_err;
        }

out_v6:
#if IS_ENABLED(CONFIG_IPV6)
        if (!brmctx->multicast_querier &&
            !timer_pending(&brmctx->ip6_other_query.timer))
                goto out;

        br_multicast_read_querier(&brmctx->ip6_querier, &querier);
        if (nla_put_in6_addr(skb, BRIDGE_QUERIER_IPV6_ADDRESS,
                             &querier.addr.src.ip6)) {
                rcu_read_unlock();
                goto out_err;
        }

        p = __br_multicast_get_querier_port(brmctx->br, &querier);
        if (timer_pending(&brmctx->ip6_other_query.timer) &&
            (nla_put_u64_64bit(skb, BRIDGE_QUERIER_IPV6_OTHER_TIMER,
                               br_timer_value(&brmctx->ip6_other_query.timer),
                               BRIDGE_QUERIER_PAD) ||
             (p && nla_put_u32(skb, BRIDGE_QUERIER_IPV6_PORT,
                               p->dev->ifindex)))) {
                rcu_read_unlock();
                goto out_err;
        }
out:
#endif
        rcu_read_unlock();
        nla_nest_end(skb, nest);
        if (!nla_len(nest))
                nla_nest_cancel(skb, nest);

        return 0;

out_err:
        nla_nest_cancel(skb, nest);
        return -EMSGSIZE;
}

static void
br_multicast_update_query_timer(struct net_bridge_mcast *brmctx,
                                struct bridge_mcast_other_query *query,
                                unsigned long max_delay)
{
        if (!timer_pending(&query->timer))
                mod_timer(&query->delay_timer, jiffies + max_delay);

        mod_timer(&query->timer, jiffies + brmctx->multicast_querier_interval);
}

static void br_port_mc_router_state_change(struct net_bridge_port *p,
                                           bool is_mc_router)
{
        struct switchdev_attr attr = {
                .orig_dev = p->dev,
                .id = SWITCHDEV_ATTR_ID_PORT_MROUTER,
                .flags = SWITCHDEV_F_DEFER,
                .u.mrouter = is_mc_router,
        };

        switchdev_port_attr_set(p->dev, &attr, NULL);
}

static struct net_bridge_port *
br_multicast_rport_from_node(struct net_bridge_mcast *brmctx,
                             struct hlist_head *mc_router_list,
                             struct hlist_node *rlist)
{
        struct net_bridge_mcast_port *pmctx;

#if IS_ENABLED(CONFIG_IPV6)
        if (mc_router_list == &brmctx->ip6_mc_router_list)
                pmctx = hlist_entry(rlist, struct net_bridge_mcast_port,
                                    ip6_rlist);
        else
#endif
                pmctx = hlist_entry(rlist, struct net_bridge_mcast_port,
                                    ip4_rlist);

        return pmctx->port;
}

static struct hlist_node *
br_multicast_get_rport_slot(struct net_bridge_mcast *brmctx,
                            struct net_bridge_port *port,
                            struct hlist_head *mc_router_list)

{
        struct hlist_node *slot = NULL;
        struct net_bridge_port *p;
        struct hlist_node *rlist;

        hlist_for_each(rlist, mc_router_list) {
                p = br_multicast_rport_from_node(brmctx, mc_router_list, rlist);

                if ((unsigned long)port >= (unsigned long)p)
                        break;

                slot = rlist;
        }

        return slot;
}

static bool br_multicast_no_router_otherpf(struct net_bridge_mcast_port *pmctx,
                                           struct hlist_node *rnode)
{
#if IS_ENABLED(CONFIG_IPV6)
        if (rnode != &pmctx->ip6_rlist)
                return hlist_unhashed(&pmctx->ip6_rlist);
        else
                return hlist_unhashed(&pmctx->ip4_rlist);
#else
        return true;
#endif
}

/* Add port to router_list
 *  list is maintained ordered by pointer value
 *  and locked by br->multicast_lock and RCU
 */
static void br_multicast_add_router(struct net_bridge_mcast *brmctx,
                                    struct net_bridge_mcast_port *pmctx,
                                    struct hlist_node *rlist,
                                    struct hlist_head *mc_router_list)
{
        struct hlist_node *slot;

        if (!hlist_unhashed(rlist))
                return;

        slot = br_multicast_get_rport_slot(brmctx, pmctx->port, mc_router_list);

        if (slot)
                hlist_add_behind_rcu(rlist, slot);
        else
                hlist_add_head_rcu(rlist, mc_router_list);

        /* For backwards compatibility for now, only notify if we
         * switched from no IPv4/IPv6 multicast router to a new
         * IPv4 or IPv6 multicast router.
         */
        if (br_multicast_no_router_otherpf(pmctx, rlist)) {
                br_rtr_notify(pmctx->port->br->dev, pmctx, RTM_NEWMDB);
                br_port_mc_router_state_change(pmctx->port, true);
        }
}

/* Add port to router_list
 *  list is maintained ordered by pointer value
 *  and locked by br->multicast_lock and RCU
 */
static void br_ip4_multicast_add_router(struct net_bridge_mcast *brmctx,
                                        struct net_bridge_mcast_port *pmctx)
{
        br_multicast_add_router(brmctx, pmctx, &pmctx->ip4_rlist,
                                &brmctx->ip4_mc_router_list);
}

/* Add port to router_list
 *  list is maintained ordered by pointer value
 *  and locked by br->multicast_lock and RCU
 */
static void br_ip6_multicast_add_router(struct net_bridge_mcast *brmctx,
                                        struct net_bridge_mcast_port *pmctx)
{
#if IS_ENABLED(CONFIG_IPV6)
        br_multicast_add_router(brmctx, pmctx, &pmctx->ip6_rlist,
                                &brmctx->ip6_mc_router_list);
#endif
}

static void br_multicast_mark_router(struct net_bridge_mcast *brmctx,
                                     struct net_bridge_mcast_port *pmctx,
                                     struct timer_list *timer,
                                     struct hlist_node *rlist,
                                     struct hlist_head *mc_router_list)
{
        unsigned long now = jiffies;

        if (!br_multicast_ctx_should_use(brmctx, pmctx))
                return;

        if (!pmctx) {
                if (brmctx->multicast_router == MDB_RTR_TYPE_TEMP_QUERY) {
                        if (!br_ip4_multicast_is_router(brmctx) &&
                            !br_ip6_multicast_is_router(brmctx))
                                br_mc_router_state_change(brmctx->br, true);
                        mod_timer(timer, now + brmctx->multicast_querier_interval);
                }
                return;
        }

        if (pmctx->multicast_router == MDB_RTR_TYPE_DISABLED ||
            pmctx->multicast_router == MDB_RTR_TYPE_PERM)
                return;

        br_multicast_add_router(brmctx, pmctx, rlist, mc_router_list);
        mod_timer(timer, now + brmctx->multicast_querier_interval);
}

static void br_ip4_multicast_mark_router(struct net_bridge_mcast *brmctx,
                                         struct net_bridge_mcast_port *pmctx)
{
        struct timer_list *timer = &brmctx->ip4_mc_router_timer;
        struct hlist_node *rlist = NULL;

        if (pmctx) {
                timer = &pmctx->ip4_mc_router_timer;
                rlist = &pmctx->ip4_rlist;
        }

        br_multicast_mark_router(brmctx, pmctx, timer, rlist,
                                 &brmctx->ip4_mc_router_list);
}

static void br_ip6_multicast_mark_router(struct net_bridge_mcast *brmctx,
                                         struct net_bridge_mcast_port *pmctx)
{
#if IS_ENABLED(CONFIG_IPV6)
        struct timer_list *timer = &brmctx->ip6_mc_router_timer;
        struct hlist_node *rlist = NULL;

        if (pmctx) {
                timer = &pmctx->ip6_mc_router_timer;
                rlist = &pmctx->ip6_rlist;
        }

        br_multicast_mark_router(brmctx, pmctx, timer, rlist,
                                 &brmctx->ip6_mc_router_list);
#endif
}

static void
br_ip4_multicast_query_received(struct net_bridge_mcast *brmctx,
                                struct net_bridge_mcast_port *pmctx,
                                struct bridge_mcast_other_query *query,
                                struct br_ip *saddr,
                                unsigned long max_delay)
{
        if (!br_multicast_select_querier(brmctx, pmctx, saddr))
                return;

        br_multicast_update_query_timer(brmctx, query, max_delay);
        br_ip4_multicast_mark_router(brmctx, pmctx);
}

#if IS_ENABLED(CONFIG_IPV6)
static void
br_ip6_multicast_query_received(struct net_bridge_mcast *brmctx,
                                struct net_bridge_mcast_port *pmctx,
                                struct bridge_mcast_other_query *query,
                                struct br_ip *saddr,
                                unsigned long max_delay)
{
        if (!br_multicast_select_querier(brmctx, pmctx, saddr))
                return;

        br_multicast_update_query_timer(brmctx, query, max_delay);
        br_ip6_multicast_mark_router(brmctx, pmctx);
}
#endif

static void br_ip4_multicast_query(struct net_bridge_mcast *brmctx,
                                   struct net_bridge_mcast_port *pmctx,
                                   struct sk_buff *skb,
                                   u16 vid)
{
        unsigned int transport_len = ip_transport_len(skb);
        const struct iphdr *iph = ip_hdr(skb);
        struct igmphdr *ih = igmp_hdr(skb);
        struct net_bridge_mdb_entry *mp;
        struct igmpv3_query *ih3;
        struct net_bridge_port_group *p;
        struct net_bridge_port_group __rcu **pp;
        struct br_ip saddr = {};
        unsigned long max_delay;
        unsigned long now = jiffies;
        __be32 group;

        spin_lock(&brmctx->br->multicast_lock);
        if (!br_multicast_ctx_should_use(brmctx, pmctx))
                goto out;

        group = ih->group;

        if (transport_len == sizeof(*ih)) {
                max_delay = ih->code * (HZ / IGMP_TIMER_SCALE);

                if (!max_delay) {
                        max_delay = 10 * HZ;
                        group = 0;
                }
        } else if (transport_len >= sizeof(*ih3)) {
                ih3 = igmpv3_query_hdr(skb);
                if (ih3->nsrcs ||
                    (brmctx->multicast_igmp_version == 3 && group &&
                     ih3->suppress))
                        goto out;

                max_delay = ih3->code ?
                            IGMPV3_MRC(ih3->code) * (HZ / IGMP_TIMER_SCALE) : 1;
        } else {
                goto out;
        }

        if (!group) {
                saddr.proto = htons(ETH_P_IP);
                saddr.src.ip4 = iph->saddr;

                br_ip4_multicast_query_received(brmctx, pmctx,
                                                &brmctx->ip4_other_query,
                                                &saddr, max_delay);
                goto out;
        }

        mp = br_mdb_ip4_get(brmctx->br, group, vid);
        if (!mp)
                goto out;

        max_delay *= brmctx->multicast_last_member_count;

        if (mp->host_joined &&
            (timer_pending(&mp->timer) ?
             time_after(mp->timer.expires, now + max_delay) :
             try_to_del_timer_sync(&mp->timer) >= 0))
                mod_timer(&mp->timer, now + max_delay);

        for (pp = &mp->ports;
             (p = mlock_dereference(*pp, brmctx->br)) != NULL;
             pp = &p->next) {
                if (timer_pending(&p->timer) ?
                    time_after(p->timer.expires, now + max_delay) :
                    try_to_del_timer_sync(&p->timer) >= 0 &&
                    (brmctx->multicast_igmp_version == 2 ||
                     p->filter_mode == MCAST_EXCLUDE))
                        mod_timer(&p->timer, now + max_delay);
        }

out:
        spin_unlock(&brmctx->br->multicast_lock);
}

#if IS_ENABLED(CONFIG_IPV6)
static int br_ip6_multicast_query(struct net_bridge_mcast *brmctx,
                                  struct net_bridge_mcast_port *pmctx,
                                  struct sk_buff *skb,
                                  u16 vid)
{
        unsigned int transport_len = ipv6_transport_len(skb);
        struct mld_msg *mld;
        struct net_bridge_mdb_entry *mp;
        struct mld2_query *mld2q;
        struct net_bridge_port_group *p;
        struct net_bridge_port_group __rcu **pp;
        struct br_ip saddr = {};
        unsigned long max_delay;
        unsigned long now = jiffies;
        unsigned int offset = skb_transport_offset(skb);
        const struct in6_addr *group = NULL;
        bool is_general_query;
        int err = 0;

        spin_lock(&brmctx->br->multicast_lock);
        if (!br_multicast_ctx_should_use(brmctx, pmctx))
                goto out;

        if (transport_len == sizeof(*mld)) {
                if (!pskb_may_pull(skb, offset + sizeof(*mld))) {
                        err = -EINVAL;
                        goto out;
                }
                mld = (struct mld_msg *) icmp6_hdr(skb);
                max_delay = msecs_to_jiffies(ntohs(mld->mld_maxdelay));
                if (max_delay)
                        group = &mld->mld_mca;
        } else {
                if (!pskb_may_pull(skb, offset + sizeof(*mld2q))) {
                        err = -EINVAL;
                        goto out;
                }
                mld2q = (struct mld2_query *)icmp6_hdr(skb);
                if (!mld2q->mld2q_nsrcs)
                        group = &mld2q->mld2q_mca;
                if (brmctx->multicast_mld_version == 2 &&
                    !ipv6_addr_any(&mld2q->mld2q_mca) &&
                    mld2q->mld2q_suppress)
                        goto out;

                max_delay = max(msecs_to_jiffies(mldv2_mrc(mld2q)), 1UL);
        }

        is_general_query = group && ipv6_addr_any(group);

        if (is_general_query) {
                saddr.proto = htons(ETH_P_IPV6);
                saddr.src.ip6 = ipv6_hdr(skb)->saddr;

                br_ip6_multicast_query_received(brmctx, pmctx,
                                                &brmctx->ip6_other_query,
                                                &saddr, max_delay);
                goto out;
        } else if (!group) {
                goto out;
        }

        mp = br_mdb_ip6_get(brmctx->br, group, vid);
        if (!mp)
                goto out;

        max_delay *= brmctx->multicast_last_member_count;
        if (mp->host_joined &&
            (timer_pending(&mp->timer) ?
             time_after(mp->timer.expires, now + max_delay) :
             try_to_del_timer_sync(&mp->timer) >= 0))
                mod_timer(&mp->timer, now + max_delay);

        for (pp = &mp->ports;
             (p = mlock_dereference(*pp, brmctx->br)) != NULL;
             pp = &p->next) {
                if (timer_pending(&p->timer) ?
                    time_after(p->timer.expires, now + max_delay) :
                    try_to_del_timer_sync(&p->timer) >= 0 &&
                    (brmctx->multicast_mld_version == 1 ||
                     p->filter_mode == MCAST_EXCLUDE))
                        mod_timer(&p->timer, now + max_delay);
        }

out:
        spin_unlock(&brmctx->br->multicast_lock);
        return err;
}
#endif

static void
br_multicast_leave_group(struct net_bridge_mcast *brmctx,
                         struct net_bridge_mcast_port *pmctx,
                         struct br_ip *group,
                         struct bridge_mcast_other_query *other_query,
                         struct bridge_mcast_own_query *own_query,
                         const unsigned char *src)
{
        struct net_bridge_mdb_entry *mp;
        struct net_bridge_port_group *p;
        unsigned long now;
        unsigned long time;

        spin_lock(&brmctx->br->multicast_lock);
        if (!br_multicast_ctx_should_use(brmctx, pmctx))
                goto out;

        mp = br_mdb_ip_get(brmctx->br, group);
        if (!mp)
                goto out;

        if (pmctx && (pmctx->port->flags & BR_MULTICAST_FAST_LEAVE)) {
                struct net_bridge_port_group __rcu **pp;

                for (pp = &mp->ports;
                     (p = mlock_dereference(*pp, brmctx->br)) != NULL;
                     pp = &p->next) {
                        if (!br_port_group_equal(p, pmctx->port, src))
                                continue;

                        if (p->flags & MDB_PG_FLAGS_PERMANENT)
                                break;

                        p->flags |= MDB_PG_FLAGS_FAST_LEAVE;
                        br_multicast_del_pg(mp, p, pp);
                }
                goto out;
        }

        if (timer_pending(&other_query->timer))
                goto out;

        if (brmctx->multicast_querier) {
                __br_multicast_send_query(brmctx, pmctx, NULL, NULL, &mp->addr,
                                          false, 0, NULL);

                time = jiffies + brmctx->multicast_last_member_count *
                                 brmctx->multicast_last_member_interval;

                mod_timer(&own_query->timer, time);

                for (p = mlock_dereference(mp->ports, brmctx->br);
                     p != NULL && pmctx != NULL;
                     p = mlock_dereference(p->next, brmctx->br)) {
                        if (!br_port_group_equal(p, pmctx->port, src))
                                continue;

                        if (!hlist_unhashed(&p->mglist) &&
                            (timer_pending(&p->timer) ?
                             time_after(p->timer.expires, time) :
                             try_to_del_timer_sync(&p->timer) >= 0)) {
                                mod_timer(&p->timer, time);
                        }

                        break;
                }
        }

        now = jiffies;
        time = now + brmctx->multicast_last_member_count *
                     brmctx->multicast_last_member_interval;

        if (!pmctx) {
                if (mp->host_joined &&
                    (timer_pending(&mp->timer) ?
                     time_after(mp->timer.expires, time) :
                     try_to_del_timer_sync(&mp->timer) >= 0)) {
                        mod_timer(&mp->timer, time);
                }

                goto out;
        }

        for (p = mlock_dereference(mp->ports, brmctx->br);
             p != NULL;
             p = mlock_dereference(p->next, brmctx->br)) {
                if (p->key.port != pmctx->port)
                        continue;

                if (!hlist_unhashed(&p->mglist) &&
                    (timer_pending(&p->timer) ?
                     time_after(p->timer.expires, time) :
                     try_to_del_timer_sync(&p->timer) >= 0)) {
                        mod_timer(&p->timer, time);
                }

                break;
        }
out:
        spin_unlock(&brmctx->br->multicast_lock);
}

static void br_ip4_multicast_leave_group(struct net_bridge_mcast *brmctx,
                                         struct net_bridge_mcast_port *pmctx,
                                         __be32 group,
                                         __u16 vid,
                                         const unsigned char *src)
{
        struct br_ip br_group;
        struct bridge_mcast_own_query *own_query;

        if (ipv4_is_local_multicast(group))
                return;

        own_query = pmctx ? &pmctx->ip4_own_query : &brmctx->ip4_own_query;

        memset(&br_group, 0, sizeof(br_group));
        br_group.dst.ip4 = group;
        br_group.proto = htons(ETH_P_IP);
        br_group.vid = vid;

        br_multicast_leave_group(brmctx, pmctx, &br_group,
                                 &brmctx->ip4_other_query,
                                 own_query, src);
}

#if IS_ENABLED(CONFIG_IPV6)
static void br_ip6_multicast_leave_group(struct net_bridge_mcast *brmctx,
                                         struct net_bridge_mcast_port *pmctx,
                                         const struct in6_addr *group,
                                         __u16 vid,
                                         const unsigned char *src)
{
        struct br_ip br_group;
        struct bridge_mcast_own_query *own_query;

        if (ipv6_addr_is_ll_all_nodes(group))
                return;

        own_query = pmctx ? &pmctx->ip6_own_query : &brmctx->ip6_own_query;

        memset(&br_group, 0, sizeof(br_group));
        br_group.dst.ip6 = *group;
        br_group.proto = htons(ETH_P_IPV6);
        br_group.vid = vid;

        br_multicast_leave_group(brmctx, pmctx, &br_group,
                                 &brmctx->ip6_other_query,
                                 own_query, src);
}
#endif

static void br_multicast_err_count(const struct net_bridge *br,
                                   const struct net_bridge_port *p,
                                   __be16 proto)
{
        struct bridge_mcast_stats __percpu *stats;
        struct bridge_mcast_stats *pstats;

        if (!br_opt_get(br, BROPT_MULTICAST_STATS_ENABLED))
                return;

        if (p)
                stats = p->mcast_stats;
        else
                stats = br->mcast_stats;
        if (WARN_ON(!stats))
                return;

        pstats = this_cpu_ptr(stats);

        u64_stats_update_begin(&pstats->syncp);
        switch (proto) {
        case htons(ETH_P_IP):
                pstats->mstats.igmp_parse_errors++;
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                pstats->mstats.mld_parse_errors++;
                break;
#endif
        }
        u64_stats_update_end(&pstats->syncp);
}

static void br_multicast_pim(struct net_bridge_mcast *brmctx,
                             struct net_bridge_mcast_port *pmctx,
                             const struct sk_buff *skb)
{
        unsigned int offset = skb_transport_offset(skb);
        struct pimhdr *pimhdr, _pimhdr;

        pimhdr = skb_header_pointer(skb, offset, sizeof(_pimhdr), &_pimhdr);
        if (!pimhdr || pim_hdr_version(pimhdr) != PIM_VERSION ||
            pim_hdr_type(pimhdr) != PIM_TYPE_HELLO)
                return;

        spin_lock(&brmctx->br->multicast_lock);
        br_ip4_multicast_mark_router(brmctx, pmctx);
        spin_unlock(&brmctx->br->multicast_lock);
}

static int br_ip4_multicast_mrd_rcv(struct net_bridge_mcast *brmctx,
                                    struct net_bridge_mcast_port *pmctx,
                                    struct sk_buff *skb)
{
        if (ip_hdr(skb)->protocol != IPPROTO_IGMP ||
            igmp_hdr(skb)->type != IGMP_MRDISC_ADV)
                return -ENOMSG;

        spin_lock(&brmctx->br->multicast_lock);
        br_ip4_multicast_mark_router(brmctx, pmctx);
        spin_unlock(&brmctx->br->multicast_lock);

        return 0;
}

static int br_multicast_ipv4_rcv(struct net_bridge_mcast *brmctx,
                                 struct net_bridge_mcast_port *pmctx,
                                 struct sk_buff *skb,
                                 u16 vid)
{
        struct net_bridge_port *p = pmctx ? pmctx->port : NULL;
        const unsigned char *src;
        struct igmphdr *ih;
        int err;

        err = ip_mc_check_igmp(skb);

        if (err == -ENOMSG) {
                if (!ipv4_is_local_multicast(ip_hdr(skb)->daddr)) {
                        BR_INPUT_SKB_CB(skb)->mrouters_only = 1;
                } else if (pim_ipv4_all_pim_routers(ip_hdr(skb)->daddr)) {
                        if (ip_hdr(skb)->protocol == IPPROTO_PIM)
                                br_multicast_pim(brmctx, pmctx, skb);
                } else if (ipv4_is_all_snoopers(ip_hdr(skb)->daddr)) {
                        br_ip4_multicast_mrd_rcv(brmctx, pmctx, skb);
                }

                return 0;
        } else if (err < 0) {
                br_multicast_err_count(brmctx->br, p, skb->protocol);
                return err;
        }

        ih = igmp_hdr(skb);
        src = eth_hdr(skb)->h_source;
        BR_INPUT_SKB_CB(skb)->igmp = ih->type;

        switch (ih->type) {
        case IGMP_HOST_MEMBERSHIP_REPORT:
        case IGMPV2_HOST_MEMBERSHIP_REPORT:
                BR_INPUT_SKB_CB(skb)->mrouters_only = 1;
                err = br_ip4_multicast_add_group(brmctx, pmctx, ih->group, vid,
                                                 src, true);
                break;
        case IGMPV3_HOST_MEMBERSHIP_REPORT:
                err = br_ip4_multicast_igmp3_report(brmctx, pmctx, skb, vid);
                break;
        case IGMP_HOST_MEMBERSHIP_QUERY:
                br_ip4_multicast_query(brmctx, pmctx, skb, vid);
                break;
        case IGMP_HOST_LEAVE_MESSAGE:
                br_ip4_multicast_leave_group(brmctx, pmctx, ih->group, vid, src);
                break;
        }

        br_multicast_count(brmctx->br, p, skb, BR_INPUT_SKB_CB(skb)->igmp,
                           BR_MCAST_DIR_RX);

        return err;
}

#if IS_ENABLED(CONFIG_IPV6)
static void br_ip6_multicast_mrd_rcv(struct net_bridge_mcast *brmctx,
                                     struct net_bridge_mcast_port *pmctx,
                                     struct sk_buff *skb)
{
        if (icmp6_hdr(skb)->icmp6_type != ICMPV6_MRDISC_ADV)
                return;

        spin_lock(&brmctx->br->multicast_lock);
        br_ip6_multicast_mark_router(brmctx, pmctx);
        spin_unlock(&brmctx->br->multicast_lock);
}

static int br_multicast_ipv6_rcv(struct net_bridge_mcast *brmctx,
                                 struct net_bridge_mcast_port *pmctx,
                                 struct sk_buff *skb,
                                 u16 vid)
{
        struct net_bridge_port *p = pmctx ? pmctx->port : NULL;
        const unsigned char *src;
        struct mld_msg *mld;
        int err;

        err = ipv6_mc_check_mld(skb);

        if (err == -ENOMSG || err == -ENODATA) {
                if (!ipv6_addr_is_ll_all_nodes(&ipv6_hdr(skb)->daddr))
                        BR_INPUT_SKB_CB(skb)->mrouters_only = 1;
                if (err == -ENODATA &&
                    ipv6_addr_is_all_snoopers(&ipv6_hdr(skb)->daddr))
                        br_ip6_multicast_mrd_rcv(brmctx, pmctx, skb);

                return 0;
        } else if (err < 0) {
                br_multicast_err_count(brmctx->br, p, skb->protocol);
                return err;
        }

        mld = (struct mld_msg *)skb_transport_header(skb);
        BR_INPUT_SKB_CB(skb)->igmp = mld->mld_type;

        switch (mld->mld_type) {
        case ICMPV6_MGM_REPORT:
                src = eth_hdr(skb)->h_source;
                BR_INPUT_SKB_CB(skb)->mrouters_only = 1;
                err = br_ip6_multicast_add_group(brmctx, pmctx, &mld->mld_mca,
                                                 vid, src, true);
                break;
        case ICMPV6_MLD2_REPORT:
                err = br_ip6_multicast_mld2_report(brmctx, pmctx, skb, vid);
                break;
        case ICMPV6_MGM_QUERY:
                err = br_ip6_multicast_query(brmctx, pmctx, skb, vid);
                break;
        case ICMPV6_MGM_REDUCTION:
                src = eth_hdr(skb)->h_source;
                br_ip6_multicast_leave_group(brmctx, pmctx, &mld->mld_mca, vid,
                                             src);
                break;
        }

        br_multicast_count(brmctx->br, p, skb, BR_INPUT_SKB_CB(skb)->igmp,
                           BR_MCAST_DIR_RX);

        return err;
}
#endif

int br_multicast_rcv(struct net_bridge_mcast **brmctx,
                     struct net_bridge_mcast_port **pmctx,
                     struct net_bridge_vlan *vlan,
                     struct sk_buff *skb, u16 vid)
{
        int ret = 0;

        BR_INPUT_SKB_CB(skb)->igmp = 0;
        BR_INPUT_SKB_CB(skb)->mrouters_only = 0;

        if (!br_opt_get((*brmctx)->br, BROPT_MULTICAST_ENABLED))
                return 0;

        if (br_opt_get((*brmctx)->br, BROPT_MCAST_VLAN_SNOOPING_ENABLED) && vlan) {
                const struct net_bridge_vlan *masterv;

                /* the vlan has the master flag set only when transmitting
                 * through the bridge device
                 */
                if (br_vlan_is_master(vlan)) {
                        masterv = vlan;
                        *brmctx = &vlan->br_mcast_ctx;
                        *pmctx = NULL;
                } else {
                        masterv = vlan->brvlan;
                        *brmctx = &vlan->brvlan->br_mcast_ctx;
                        *pmctx = &vlan->port_mcast_ctx;
                }

                if (!(masterv->priv_flags & BR_VLFLAG_GLOBAL_MCAST_ENABLED))
                        return 0;
        }

        switch (skb->protocol) {
        case htons(ETH_P_IP):
                ret = br_multicast_ipv4_rcv(*brmctx, *pmctx, skb, vid);
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                ret = br_multicast_ipv6_rcv(*brmctx, *pmctx, skb, vid);
                break;
#endif
        }

        return ret;
}

static void br_multicast_query_expired(struct net_bridge_mcast *brmctx,
                                       struct bridge_mcast_own_query *query,
                                       struct bridge_mcast_querier *querier)
{
        spin_lock(&brmctx->br->multicast_lock);
        if (br_multicast_ctx_vlan_disabled(brmctx))
                goto out;

        if (query->startup_sent < brmctx->multicast_startup_query_count)
                query->startup_sent++;

        br_multicast_send_query(brmctx, NULL, query);
out:
        spin_unlock(&brmctx->br->multicast_lock);
}

static void br_ip4_multicast_query_expired(struct timer_list *t)
{
        struct net_bridge_mcast *brmctx = from_timer(brmctx, t,
                                                     ip4_own_query.timer);

        br_multicast_query_expired(brmctx, &brmctx->ip4_own_query,
                                   &brmctx->ip4_querier);
}

#if IS_ENABLED(CONFIG_IPV6)
static void br_ip6_multicast_query_expired(struct timer_list *t)
{
        struct net_bridge_mcast *brmctx = from_timer(brmctx, t,
                                                     ip6_own_query.timer);

        br_multicast_query_expired(brmctx, &brmctx->ip6_own_query,
                                   &brmctx->ip6_querier);
}
#endif

static void br_multicast_gc_work(struct work_struct *work)
{
        struct net_bridge *br = container_of(work, struct net_bridge,
                                             mcast_gc_work);
        HLIST_HEAD(deleted_head);

        spin_lock_bh(&br->multicast_lock);
        hlist_move_list(&br->mcast_gc_list, &deleted_head);
        spin_unlock_bh(&br->multicast_lock);

        br_multicast_gc(&deleted_head);
}

void br_multicast_ctx_init(struct net_bridge *br,
                           struct net_bridge_vlan *vlan,
                           struct net_bridge_mcast *brmctx)
{
        brmctx->br = br;
        brmctx->vlan = vlan;
        brmctx->multicast_router = MDB_RTR_TYPE_TEMP_QUERY;
        brmctx->multicast_last_member_count = 2;
        brmctx->multicast_startup_query_count = 2;

        brmctx->multicast_last_member_interval = HZ;
        brmctx->multicast_query_response_interval = 10 * HZ;
        brmctx->multicast_startup_query_interval = 125 * HZ / 4;
        brmctx->multicast_query_interval = 125 * HZ;
        brmctx->multicast_querier_interval = 255 * HZ;
        brmctx->multicast_membership_interval = 260 * HZ;

        brmctx->ip4_querier.port_ifidx = 0;
        seqcount_spinlock_init(&brmctx->ip4_querier.seq, &br->multicast_lock);
        brmctx->multicast_igmp_version = 2;
#if IS_ENABLED(CONFIG_IPV6)
        brmctx->multicast_mld_version = 1;
        brmctx->ip6_querier.port_ifidx = 0;
        seqcount_spinlock_init(&brmctx->ip6_querier.seq, &br->multicast_lock);
#endif

        timer_setup(&brmctx->ip4_mc_router_timer,
                    br_ip4_multicast_local_router_expired, 0);
        timer_setup(&brmctx->ip4_other_query.timer,
                    br_ip4_multicast_querier_expired, 0);
        timer_setup(&brmctx->ip4_other_query.delay_timer,
                    br_multicast_query_delay_expired, 0);
        timer_setup(&brmctx->ip4_own_query.timer,
                    br_ip4_multicast_query_expired, 0);
#if IS_ENABLED(CONFIG_IPV6)
        timer_setup(&brmctx->ip6_mc_router_timer,
                    br_ip6_multicast_local_router_expired, 0);
        timer_setup(&brmctx->ip6_other_query.timer,
                    br_ip6_multicast_querier_expired, 0);
        timer_setup(&brmctx->ip6_other_query.delay_timer,
                    br_multicast_query_delay_expired, 0);
        timer_setup(&brmctx->ip6_own_query.timer,
                    br_ip6_multicast_query_expired, 0);
#endif
}

void br_multicast_ctx_deinit(struct net_bridge_mcast *brmctx)
{
        __br_multicast_stop(brmctx);
}

void br_multicast_init(struct net_bridge *br)
{
        br->hash_max = BR_MULTICAST_DEFAULT_HASH_MAX;

        br_multicast_ctx_init(br, NULL, &br->multicast_ctx);

        br_opt_toggle(br, BROPT_MULTICAST_ENABLED, true);
        br_opt_toggle(br, BROPT_HAS_IPV6_ADDR, true);

        spin_lock_init(&br->multicast_lock);
        INIT_HLIST_HEAD(&br->mdb_list);
        INIT_HLIST_HEAD(&br->mcast_gc_list);
        INIT_WORK(&br->mcast_gc_work, br_multicast_gc_work);
}

static void br_ip4_multicast_join_snoopers(struct net_bridge *br)
{
        struct in_device *in_dev = in_dev_get(br->dev);

        if (!in_dev)
                return;

        __ip_mc_inc_group(in_dev, htonl(INADDR_ALLSNOOPERS_GROUP), GFP_ATOMIC);
        in_dev_put(in_dev);
}

#if IS_ENABLED(CONFIG_IPV6)
static void br_ip6_multicast_join_snoopers(struct net_bridge *br)
{
        struct in6_addr addr;

        ipv6_addr_set(&addr, htonl(0xff020000), 0, 0, htonl(0x6a));
        ipv6_dev_mc_inc(br->dev, &addr);
}
#else
static inline void br_ip6_multicast_join_snoopers(struct net_bridge *br)
{
}
#endif

void br_multicast_join_snoopers(struct net_bridge *br)
{
        br_ip4_multicast_join_snoopers(br);
        br_ip6_multicast_join_snoopers(br);
}

static void br_ip4_multicast_leave_snoopers(struct net_bridge *br)
{
        struct in_device *in_dev = in_dev_get(br->dev);

        if (WARN_ON(!in_dev))
                return;

        __ip_mc_dec_group(in_dev, htonl(INADDR_ALLSNOOPERS_GROUP), GFP_ATOMIC);
        in_dev_put(in_dev);
}

#if IS_ENABLED(CONFIG_IPV6)
static void br_ip6_multicast_leave_snoopers(struct net_bridge *br)
{
        struct in6_addr addr;

        ipv6_addr_set(&addr, htonl(0xff020000), 0, 0, htonl(0x6a));
        ipv6_dev_mc_dec(br->dev, &addr);
}
#else
static inline void br_ip6_multicast_leave_snoopers(struct net_bridge *br)
{
}
#endif

void br_multicast_leave_snoopers(struct net_bridge *br)
{
        br_ip4_multicast_leave_snoopers(br);
        br_ip6_multicast_leave_snoopers(br);
}

static void __br_multicast_open_query(struct net_bridge *br,
                                      struct bridge_mcast_own_query *query)
{
        query->startup_sent = 0;

        if (!br_opt_get(br, BROPT_MULTICAST_ENABLED))
                return;

        mod_timer(&query->timer, jiffies);
}

static void __br_multicast_open(struct net_bridge_mcast *brmctx)
{
        __br_multicast_open_query(brmctx->br, &brmctx->ip4_own_query);
#if IS_ENABLED(CONFIG_IPV6)
        __br_multicast_open_query(brmctx->br, &brmctx->ip6_own_query);
#endif
}

void br_multicast_open(struct net_bridge *br)
{
        ASSERT_RTNL();

        if (br_opt_get(br, BROPT_MCAST_VLAN_SNOOPING_ENABLED)) {
                struct net_bridge_vlan_group *vg;
                struct net_bridge_vlan *vlan;

                vg = br_vlan_group(br);
                if (vg) {
                        list_for_each_entry(vlan, &vg->vlan_list, vlist) {
                                struct net_bridge_mcast *brmctx;

                                brmctx = &vlan->br_mcast_ctx;
                                if (br_vlan_is_brentry(vlan) &&
                                    !br_multicast_ctx_vlan_disabled(brmctx))
                                        __br_multicast_open(&vlan->br_mcast_ctx);
                        }
                }
        } else {
                __br_multicast_open(&br->multicast_ctx);
        }
}

static void __br_multicast_stop(struct net_bridge_mcast *brmctx)
{
        del_timer_sync(&brmctx->ip4_mc_router_timer);
        del_timer_sync(&brmctx->ip4_other_query.timer);
        del_timer_sync(&brmctx->ip4_other_query.delay_timer);
        del_timer_sync(&brmctx->ip4_own_query.timer);
#if IS_ENABLED(CONFIG_IPV6)
        del_timer_sync(&brmctx->ip6_mc_router_timer);
        del_timer_sync(&brmctx->ip6_other_query.timer);
        del_timer_sync(&brmctx->ip6_other_query.delay_timer);
        del_timer_sync(&brmctx->ip6_own_query.timer);
#endif
}

void br_multicast_toggle_one_vlan(struct net_bridge_vlan *vlan, bool on)
{
        struct net_bridge *br;

        /* it's okay to check for the flag without the multicast lock because it
         * can only change under RTNL -> multicast_lock, we need the latter to
         * sync with timers and packets
         */
        if (on == !!(vlan->priv_flags & BR_VLFLAG_MCAST_ENABLED))
                return;

        if (br_vlan_is_master(vlan)) {
                br = vlan->br;

                if (!br_vlan_is_brentry(vlan) ||
                    (on &&
                     br_multicast_ctx_vlan_global_disabled(&vlan->br_mcast_ctx)))
                        return;

                spin_lock_bh(&br->multicast_lock);
                vlan->priv_flags ^= BR_VLFLAG_MCAST_ENABLED;
                spin_unlock_bh(&br->multicast_lock);

                if (on)
                        __br_multicast_open(&vlan->br_mcast_ctx);
                else
                        __br_multicast_stop(&vlan->br_mcast_ctx);
        } else {
                struct net_bridge_mcast *brmctx;

                brmctx = br_multicast_port_ctx_get_global(&vlan->port_mcast_ctx);
                if (on && br_multicast_ctx_vlan_global_disabled(brmctx))
                        return;

                br = vlan->port->br;
                spin_lock_bh(&br->multicast_lock);
                vlan->priv_flags ^= BR_VLFLAG_MCAST_ENABLED;
                if (on)
                        __br_multicast_enable_port_ctx(&vlan->port_mcast_ctx);
                else
                        __br_multicast_disable_port_ctx(&vlan->port_mcast_ctx);
                spin_unlock_bh(&br->multicast_lock);
        }
}

static void br_multicast_toggle_vlan(struct net_bridge_vlan *vlan, bool on)
{
        struct net_bridge_port *p;

        if (WARN_ON_ONCE(!br_vlan_is_master(vlan)))
                return;

        list_for_each_entry(p, &vlan->br->port_list, list) {
                struct net_bridge_vlan *vport;

                vport = br_vlan_find(nbp_vlan_group(p), vlan->vid);
                if (!vport)
                        continue;
                br_multicast_toggle_one_vlan(vport, on);
        }

        if (br_vlan_is_brentry(vlan))
                br_multicast_toggle_one_vlan(vlan, on);
}

int br_multicast_toggle_vlan_snooping(struct net_bridge *br, bool on,
                                      struct netlink_ext_ack *extack)
{
        struct net_bridge_vlan_group *vg;
        struct net_bridge_vlan *vlan;
        struct net_bridge_port *p;

        if (br_opt_get(br, BROPT_MCAST_VLAN_SNOOPING_ENABLED) == on)
                return 0;

        if (on && !br_opt_get(br, BROPT_VLAN_ENABLED)) {
                NL_SET_ERR_MSG_MOD(extack, "Cannot enable multicast vlan snooping with vlan filtering disabled");
                return -EINVAL;
        }

        vg = br_vlan_group(br);
        if (!vg)
                return 0;

        br_opt_toggle(br, BROPT_MCAST_VLAN_SNOOPING_ENABLED, on);

        /* disable/enable non-vlan mcast contexts based on vlan snooping */
        if (on)
                __br_multicast_stop(&br->multicast_ctx);
        else
                __br_multicast_open(&br->multicast_ctx);
        list_for_each_entry(p, &br->port_list, list) {
                if (on)
                        br_multicast_disable_port(p);
                else
                        br_multicast_enable_port(p);
        }

        list_for_each_entry(vlan, &vg->vlan_list, vlist)
                br_multicast_toggle_vlan(vlan, on);

        return 0;
}

bool br_multicast_toggle_global_vlan(struct net_bridge_vlan *vlan, bool on)
{
        ASSERT_RTNL();

        /* BR_VLFLAG_GLOBAL_MCAST_ENABLED relies on eventual consistency and
         * requires only RTNL to change
         */
        if (on == !!(vlan->priv_flags & BR_VLFLAG_GLOBAL_MCAST_ENABLED))
                return false;

        vlan->priv_flags ^= BR_VLFLAG_GLOBAL_MCAST_ENABLED;
        br_multicast_toggle_vlan(vlan, on);

        return true;
}

void br_multicast_stop(struct net_bridge *br)
{
        ASSERT_RTNL();

        if (br_opt_get(br, BROPT_MCAST_VLAN_SNOOPING_ENABLED)) {
                struct net_bridge_vlan_group *vg;
                struct net_bridge_vlan *vlan;

                vg = br_vlan_group(br);
                if (vg) {
                        list_for_each_entry(vlan, &vg->vlan_list, vlist) {
                                struct net_bridge_mcast *brmctx;

                                brmctx = &vlan->br_mcast_ctx;
                                if (br_vlan_is_brentry(vlan) &&
                                    !br_multicast_ctx_vlan_disabled(brmctx))
                                        __br_multicast_stop(&vlan->br_mcast_ctx);
                        }
                }
        } else {
                __br_multicast_stop(&br->multicast_ctx);
        }
}

void br_multicast_dev_del(struct net_bridge *br)
{
        struct net_bridge_mdb_entry *mp;
        HLIST_HEAD(deleted_head);
        struct hlist_node *tmp;

        spin_lock_bh(&br->multicast_lock);
        hlist_for_each_entry_safe(mp, tmp, &br->mdb_list, mdb_node)
                br_multicast_del_mdb_entry(mp);
        hlist_move_list(&br->mcast_gc_list, &deleted_head);
        spin_unlock_bh(&br->multicast_lock);

        br_multicast_ctx_deinit(&br->multicast_ctx);
        br_multicast_gc(&deleted_head);
        cancel_work_sync(&br->mcast_gc_work);

        rcu_barrier();
}

int br_multicast_set_router(struct net_bridge_mcast *brmctx, unsigned long val)
{
        int err = -EINVAL;

        spin_lock_bh(&brmctx->br->multicast_lock);

        switch (val) {
        case MDB_RTR_TYPE_DISABLED:
        case MDB_RTR_TYPE_PERM:
                br_mc_router_state_change(brmctx->br, val == MDB_RTR_TYPE_PERM);
                del_timer(&brmctx->ip4_mc_router_timer);
#if IS_ENABLED(CONFIG_IPV6)
                del_timer(&brmctx->ip6_mc_router_timer);
#endif
                brmctx->multicast_router = val;
                err = 0;
                break;
        case MDB_RTR_TYPE_TEMP_QUERY:
                if (brmctx->multicast_router != MDB_RTR_TYPE_TEMP_QUERY)
                        br_mc_router_state_change(brmctx->br, false);
                brmctx->multicast_router = val;
                err = 0;
                break;
        }

        spin_unlock_bh(&brmctx->br->multicast_lock);

        return err;
}

static void
br_multicast_rport_del_notify(struct net_bridge_mcast_port *pmctx, bool deleted)
{
        if (!deleted)
                return;

        /* For backwards compatibility for now, only notify if there is
         * no multicast router anymore for both IPv4 and IPv6.
         */
        if (!hlist_unhashed(&pmctx->ip4_rlist))
                return;
#if IS_ENABLED(CONFIG_IPV6)
        if (!hlist_unhashed(&pmctx->ip6_rlist))
                return;
#endif

        br_rtr_notify(pmctx->port->br->dev, pmctx, RTM_DELMDB);
        br_port_mc_router_state_change(pmctx->port, false);

        /* don't allow timer refresh */
        if (pmctx->multicast_router == MDB_RTR_TYPE_TEMP)
                pmctx->multicast_router = MDB_RTR_TYPE_TEMP_QUERY;
}

int br_multicast_set_port_router(struct net_bridge_mcast_port *pmctx,
                                 unsigned long val)
{
        struct net_bridge_mcast *brmctx;
        unsigned long now = jiffies;
        int err = -EINVAL;
        bool del = false;

        brmctx = br_multicast_port_ctx_get_global(pmctx);
        spin_lock_bh(&brmctx->br->multicast_lock);
        if (pmctx->multicast_router == val) {
                /* Refresh the temp router port timer */
                if (pmctx->multicast_router == MDB_RTR_TYPE_TEMP) {
                        mod_timer(&pmctx->ip4_mc_router_timer,
                                  now + brmctx->multicast_querier_interval);
#if IS_ENABLED(CONFIG_IPV6)
                        mod_timer(&pmctx->ip6_mc_router_timer,
                                  now + brmctx->multicast_querier_interval);
#endif
                }
                err = 0;
                goto unlock;
        }
        switch (val) {
        case MDB_RTR_TYPE_DISABLED:
                pmctx->multicast_router = MDB_RTR_TYPE_DISABLED;
                del |= br_ip4_multicast_rport_del(pmctx);
                del_timer(&pmctx->ip4_mc_router_timer);
                del |= br_ip6_multicast_rport_del(pmctx);
#if IS_ENABLED(CONFIG_IPV6)
                del_timer(&pmctx->ip6_mc_router_timer);
#endif
                br_multicast_rport_del_notify(pmctx, del);
                break;
        case MDB_RTR_TYPE_TEMP_QUERY:
                pmctx->multicast_router = MDB_RTR_TYPE_TEMP_QUERY;
                del |= br_ip4_multicast_rport_del(pmctx);
                del |= br_ip6_multicast_rport_del(pmctx);
                br_multicast_rport_del_notify(pmctx, del);
                break;
        case MDB_RTR_TYPE_PERM:
                pmctx->multicast_router = MDB_RTR_TYPE_PERM;
                del_timer(&pmctx->ip4_mc_router_timer);
                br_ip4_multicast_add_router(brmctx, pmctx);
#if IS_ENABLED(CONFIG_IPV6)
                del_timer(&pmctx->ip6_mc_router_timer);
#endif
                br_ip6_multicast_add_router(brmctx, pmctx);
                break;
        case MDB_RTR_TYPE_TEMP:
                pmctx->multicast_router = MDB_RTR_TYPE_TEMP;
                br_ip4_multicast_mark_router(brmctx, pmctx);
                br_ip6_multicast_mark_router(brmctx, pmctx);
                break;
        default:
                goto unlock;
        }
        err = 0;
unlock:
        spin_unlock_bh(&brmctx->br->multicast_lock);

        return err;
}

int br_multicast_set_vlan_router(struct net_bridge_vlan *v, u8 mcast_router)
{
        int err;

        if (br_vlan_is_master(v))
                err = br_multicast_set_router(&v->br_mcast_ctx, mcast_router);
        else
                err = br_multicast_set_port_router(&v->port_mcast_ctx,
                                                   mcast_router);

        return err;
}

static void br_multicast_start_querier(struct net_bridge_mcast *brmctx,
                                       struct bridge_mcast_own_query *query)
{
        struct net_bridge_port *port;

        if (!br_multicast_ctx_matches_vlan_snooping(brmctx))
                return;

        __br_multicast_open_query(brmctx->br, query);

        rcu_read_lock();
        list_for_each_entry_rcu(port, &brmctx->br->port_list, list) {
                struct bridge_mcast_own_query *ip4_own_query;
#if IS_ENABLED(CONFIG_IPV6)
                struct bridge_mcast_own_query *ip6_own_query;
#endif

                if (br_multicast_port_ctx_state_stopped(&port->multicast_ctx))
                        continue;

                if (br_multicast_ctx_is_vlan(brmctx)) {
                        struct net_bridge_vlan *vlan;

                        vlan = br_vlan_find(nbp_vlan_group_rcu(port),
                                            brmctx->vlan->vid);
                        if (!vlan ||
                            br_multicast_port_ctx_state_stopped(&vlan->port_mcast_ctx))
                                continue;

                        ip4_own_query = &vlan->port_mcast_ctx.ip4_own_query;
#if IS_ENABLED(CONFIG_IPV6)
                        ip6_own_query = &vlan->port_mcast_ctx.ip6_own_query;
#endif
                } else {
                        ip4_own_query = &port->multicast_ctx.ip4_own_query;
#if IS_ENABLED(CONFIG_IPV6)
                        ip6_own_query = &port->multicast_ctx.ip6_own_query;
#endif
                }

                if (query == &brmctx->ip4_own_query)
                        br_multicast_enable(ip4_own_query);
#if IS_ENABLED(CONFIG_IPV6)
                else
                        br_multicast_enable(ip6_own_query);
#endif
        }
        rcu_read_unlock();
}

int br_multicast_toggle(struct net_bridge *br, unsigned long val,
                        struct netlink_ext_ack *extack)
{
        struct net_bridge_port *port;
        bool change_snoopers = false;
        int err = 0;

        spin_lock_bh(&br->multicast_lock);
        if (!!br_opt_get(br, BROPT_MULTICAST_ENABLED) == !!val)
                goto unlock;

        err = br_mc_disabled_update(br->dev, val, extack);
        if (err == -EOPNOTSUPP)
                err = 0;
        if (err)
                goto unlock;

        br_opt_toggle(br, BROPT_MULTICAST_ENABLED, !!val);
        if (!br_opt_get(br, BROPT_MULTICAST_ENABLED)) {
                change_snoopers = true;
                goto unlock;
        }

        if (!netif_running(br->dev))
                goto unlock;

        br_multicast_open(br);
        list_for_each_entry(port, &br->port_list, list)
                __br_multicast_enable_port_ctx(&port->multicast_ctx);

        change_snoopers = true;

unlock:
        spin_unlock_bh(&br->multicast_lock);

        /* br_multicast_join_snoopers has the potential to cause
         * an MLD Report/Leave to be delivered to br_multicast_rcv,
         * which would in turn call br_multicast_add_group, which would
         * attempt to acquire multicast_lock. This function should be
         * called after the lock has been released to avoid deadlocks on
         * multicast_lock.
         *
         * br_multicast_leave_snoopers does not have the problem since
         * br_multicast_rcv first checks BROPT_MULTICAST_ENABLED, and
         * returns without calling br_multicast_ipv4/6_rcv if it's not
         * enabled. Moved both functions out just for symmetry.
         */
        if (change_snoopers) {
                if (br_opt_get(br, BROPT_MULTICAST_ENABLED))
                        br_multicast_join_snoopers(br);
                else
                        br_multicast_leave_snoopers(br);
        }

        return err;
}

bool br_multicast_enabled(const struct net_device *dev)
{
        struct net_bridge *br = netdev_priv(dev);

        return !!br_opt_get(br, BROPT_MULTICAST_ENABLED);
}
EXPORT_SYMBOL_GPL(br_multicast_enabled);

bool br_multicast_router(const struct net_device *dev)
{
        struct net_bridge *br = netdev_priv(dev);
        bool is_router;

        spin_lock_bh(&br->multicast_lock);
        is_router = br_multicast_is_router(&br->multicast_ctx, NULL);
        spin_unlock_bh(&br->multicast_lock);
        return is_router;
}
EXPORT_SYMBOL_GPL(br_multicast_router);

int br_multicast_set_querier(struct net_bridge_mcast *brmctx, unsigned long val)
{
        unsigned long max_delay;

        val = !!val;

        spin_lock_bh(&brmctx->br->multicast_lock);
        if (brmctx->multicast_querier == val)
                goto unlock;

        WRITE_ONCE(brmctx->multicast_querier, val);
        if (!val)
                goto unlock;

        max_delay = brmctx->multicast_query_response_interval;

        if (!timer_pending(&brmctx->ip4_other_query.timer))
                mod_timer(&brmctx->ip4_other_query.delay_timer,
                          jiffies + max_delay);

        br_multicast_start_querier(brmctx, &brmctx->ip4_own_query);

#if IS_ENABLED(CONFIG_IPV6)
        if (!timer_pending(&brmctx->ip6_other_query.timer))
                mod_timer(&brmctx->ip6_other_query.delay_timer,
                          jiffies + max_delay);

        br_multicast_start_querier(brmctx, &brmctx->ip6_own_query);
#endif

unlock:
        spin_unlock_bh(&brmctx->br->multicast_lock);

        return 0;
}

int br_multicast_set_igmp_version(struct net_bridge_mcast *brmctx,
                                  unsigned long val)
{
        /* Currently we support only version 2 and 3 */
        switch (val) {
        case 2:
        case 3:
                break;
        default:
                return -EINVAL;
        }

        spin_lock_bh(&brmctx->br->multicast_lock);
        brmctx->multicast_igmp_version = val;
        spin_unlock_bh(&brmctx->br->multicast_lock);

        return 0;
}

#if IS_ENABLED(CONFIG_IPV6)
int br_multicast_set_mld_version(struct net_bridge_mcast *brmctx,
                                 unsigned long val)
{
        /* Currently we support version 1 and 2 */
        switch (val) {
        case 1:
        case 2:
                break;
        default:
                return -EINVAL;
        }

        spin_lock_bh(&brmctx->br->multicast_lock);
        brmctx->multicast_mld_version = val;
        spin_unlock_bh(&brmctx->br->multicast_lock);

        return 0;
}
#endif

void br_multicast_set_query_intvl(struct net_bridge_mcast *brmctx,
                                  unsigned long val)
{
        unsigned long intvl_jiffies = clock_t_to_jiffies(val);

        if (intvl_jiffies < BR_MULTICAST_QUERY_INTVL_MIN) {
                br_info(brmctx->br,
                        "trying to set multicast query interval below minimum, setting to %lu (%ums)\n",
                        jiffies_to_clock_t(BR_MULTICAST_QUERY_INTVL_MIN),
                        jiffies_to_msecs(BR_MULTICAST_QUERY_INTVL_MIN));
                intvl_jiffies = BR_MULTICAST_QUERY_INTVL_MIN;
        }

        brmctx->multicast_query_interval = intvl_jiffies;
}

void br_multicast_set_startup_query_intvl(struct net_bridge_mcast *brmctx,
                                          unsigned long val)
{
        unsigned long intvl_jiffies = clock_t_to_jiffies(val);

        if (intvl_jiffies < BR_MULTICAST_STARTUP_QUERY_INTVL_MIN) {
                br_info(brmctx->br,
                        "trying to set multicast startup query interval below minimum, setting to %lu (%ums)\n",
                        jiffies_to_clock_t(BR_MULTICAST_STARTUP_QUERY_INTVL_MIN),
                        jiffies_to_msecs(BR_MULTICAST_STARTUP_QUERY_INTVL_MIN));
                intvl_jiffies = BR_MULTICAST_STARTUP_QUERY_INTVL_MIN;
        }

        brmctx->multicast_startup_query_interval = intvl_jiffies;
}

/**
 * br_multicast_list_adjacent - Returns snooped multicast addresses
 * @dev:        The bridge port adjacent to which to retrieve addresses
 * @br_ip_list:        The list to store found, snooped multicast IP addresses in
 *
 * Creates a list of IP addresses (struct br_ip_list) sensed by the multicast
 * snooping feature on all bridge ports of dev's bridge device, excluding
 * the addresses from dev itself.
 *
 * Returns the number of items added to br_ip_list.
 *
 * Notes:
 * - br_ip_list needs to be initialized by caller
 * - br_ip_list might contain duplicates in the end
 *   (needs to be taken care of by caller)
 * - br_ip_list needs to be freed by caller
 */
int br_multicast_list_adjacent(struct net_device *dev,
                               struct list_head *br_ip_list)
{
        struct net_bridge *br;
        struct net_bridge_port *port;
        struct net_bridge_port_group *group;
        struct br_ip_list *entry;
        int count = 0;

        rcu_read_lock();
        if (!br_ip_list || !netif_is_bridge_port(dev))
                goto unlock;

        port = br_port_get_rcu(dev);
        if (!port || !port->br)
                goto unlock;

        br = port->br;

        list_for_each_entry_rcu(port, &br->port_list, list) {
                if (!port->dev || port->dev == dev)
                        continue;

                hlist_for_each_entry_rcu(group, &port->mglist, mglist) {
                        entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
                        if (!entry)
                                goto unlock;

                        entry->addr = group->key.addr;
                        list_add(&entry->list, br_ip_list);
                        count++;
                }
        }

unlock:
        rcu_read_unlock();
        return count;
}
EXPORT_SYMBOL_GPL(br_multicast_list_adjacent);

/**
 * br_multicast_has_querier_anywhere - Checks for a querier on a bridge
 * @dev: The bridge port providing the bridge on which to check for a querier
 * @proto: The protocol family to check for: IGMP -> ETH_P_IP, MLD -> ETH_P_IPV6
 *
 * Checks whether the given interface has a bridge on top and if so returns
 * true if a valid querier exists anywhere on the bridged link layer.
 * Otherwise returns false.
 */
bool br_multicast_has_querier_anywhere(struct net_device *dev, int proto)
{
        struct net_bridge *br;
        struct net_bridge_port *port;
        struct ethhdr eth;
        bool ret = false;

        rcu_read_lock();
        if (!netif_is_bridge_port(dev))
                goto unlock;

        port = br_port_get_rcu(dev);
        if (!port || !port->br)
                goto unlock;

        br = port->br;

        memset(&eth, 0, sizeof(eth));
        eth.h_proto = htons(proto);

        ret = br_multicast_querier_exists(&br->multicast_ctx, &eth, NULL);

unlock:
        rcu_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(br_multicast_has_querier_anywhere);

/**
 * br_multicast_has_querier_adjacent - Checks for a querier behind a bridge port
 * @dev: The bridge port adjacent to which to check for a querier
 * @proto: The protocol family to check for: IGMP -> ETH_P_IP, MLD -> ETH_P_IPV6
 *
 * Checks whether the given interface has a bridge on top and if so returns
 * true if a selected querier is behind one of the other ports of this
 * bridge. Otherwise returns false.
 */
bool br_multicast_has_querier_adjacent(struct net_device *dev, int proto)
{
        struct net_bridge_mcast *brmctx;
        struct net_bridge *br;
        struct net_bridge_port *port;
        bool ret = false;
        int port_ifidx;

        rcu_read_lock();
        if (!netif_is_bridge_port(dev))
                goto unlock;

        port = br_port_get_rcu(dev);
        if (!port || !port->br)
                goto unlock;

        br = port->br;
        brmctx = &br->multicast_ctx;

        switch (proto) {
        case ETH_P_IP:
                port_ifidx = brmctx->ip4_querier.port_ifidx;
                if (!timer_pending(&brmctx->ip4_other_query.timer) ||
                    port_ifidx == port->dev->ifindex)
                        goto unlock;
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case ETH_P_IPV6:
                port_ifidx = brmctx->ip6_querier.port_ifidx;
                if (!timer_pending(&brmctx->ip6_other_query.timer) ||
                    port_ifidx == port->dev->ifindex)
                        goto unlock;
                break;
#endif
        default:
                goto unlock;
        }

        ret = true;
unlock:
        rcu_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(br_multicast_has_querier_adjacent);

/**
 * br_multicast_has_router_adjacent - Checks for a router behind a bridge port
 * @dev: The bridge port adjacent to which to check for a multicast router
 * @proto: The protocol family to check for: IGMP -> ETH_P_IP, MLD -> ETH_P_IPV6
 *
 * Checks whether the given interface has a bridge on top and if so returns
 * true if a multicast router is behind one of the other ports of this
 * bridge. Otherwise returns false.
 */
bool br_multicast_has_router_adjacent(struct net_device *dev, int proto)
{
        struct net_bridge_mcast_port *pmctx;
        struct net_bridge_mcast *brmctx;
        struct net_bridge_port *port;
        bool ret = false;

        rcu_read_lock();
        port = br_port_get_check_rcu(dev);
        if (!port)
                goto unlock;

        brmctx = &port->br->multicast_ctx;
        switch (proto) {
        case ETH_P_IP:
                hlist_for_each_entry_rcu(pmctx, &brmctx->ip4_mc_router_list,
                                         ip4_rlist) {
                        if (pmctx->port == port)
                                continue;

                        ret = true;
                        goto unlock;
                }
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case ETH_P_IPV6:
                hlist_for_each_entry_rcu(pmctx, &brmctx->ip6_mc_router_list,
                                         ip6_rlist) {
                        if (pmctx->port == port)
                                continue;

                        ret = true;
                        goto unlock;
                }
                break;
#endif
        default:
                /* when compiled without IPv6 support, be conservative and
                 * always assume presence of an IPv6 multicast router
                 */
                ret = true;
        }

unlock:
        rcu_read_unlock();
        return ret;
}
EXPORT_SYMBOL_GPL(br_multicast_has_router_adjacent);

static void br_mcast_stats_add(struct bridge_mcast_stats __percpu *stats,
                               const struct sk_buff *skb, u8 type, u8 dir)
{
        struct bridge_mcast_stats *pstats = this_cpu_ptr(stats);
        __be16 proto = skb->protocol;
        unsigned int t_len;

        u64_stats_update_begin(&pstats->syncp);
        switch (proto) {
        case htons(ETH_P_IP):
                t_len = ntohs(ip_hdr(skb)->tot_len) - ip_hdrlen(skb);
                switch (type) {
                case IGMP_HOST_MEMBERSHIP_REPORT:
                        pstats->mstats.igmp_v1reports[dir]++;
                        break;
                case IGMPV2_HOST_MEMBERSHIP_REPORT:
                        pstats->mstats.igmp_v2reports[dir]++;
                        break;
                case IGMPV3_HOST_MEMBERSHIP_REPORT:
                        pstats->mstats.igmp_v3reports[dir]++;
                        break;
                case IGMP_HOST_MEMBERSHIP_QUERY:
                        if (t_len != sizeof(struct igmphdr)) {
                                pstats->mstats.igmp_v3queries[dir]++;
                        } else {
                                unsigned int offset = skb_transport_offset(skb);
                                struct igmphdr *ih, _ihdr;

                                ih = skb_header_pointer(skb, offset,
                                                        sizeof(_ihdr), &_ihdr);
                                if (!ih)
                                        break;
                                if (!ih->code)
                                        pstats->mstats.igmp_v1queries[dir]++;
                                else
                                        pstats->mstats.igmp_v2queries[dir]++;
                        }
                        break;
                case IGMP_HOST_LEAVE_MESSAGE:
                        pstats->mstats.igmp_leaves[dir]++;
                        break;
                }
                break;
#if IS_ENABLED(CONFIG_IPV6)
        case htons(ETH_P_IPV6):
                t_len = ntohs(ipv6_hdr(skb)->payload_len) +
                        sizeof(struct ipv6hdr);
                t_len -= skb_network_header_len(skb);
                switch (type) {
                case ICMPV6_MGM_REPORT:
                        pstats->mstats.mld_v1reports[dir]++;
                        break;
                case ICMPV6_MLD2_REPORT:
                        pstats->mstats.mld_v2reports[dir]++;
                        break;
                case ICMPV6_MGM_QUERY:
                        if (t_len != sizeof(struct mld_msg))
                                pstats->mstats.mld_v2queries[dir]++;
                        else
                                pstats->mstats.mld_v1queries[dir]++;
                        break;
                case ICMPV6_MGM_REDUCTION:
                        pstats->mstats.mld_leaves[dir]++;
                        break;
                }
                break;
#endif /* CONFIG_IPV6 */
        }
        u64_stats_update_end(&pstats->syncp);
}

void br_multicast_count(struct net_bridge *br,
                        const struct net_bridge_port *p,
                        const struct sk_buff *skb, u8 type, u8 dir)
{
        struct bridge_mcast_stats __percpu *stats;

        /* if multicast_disabled is true then igmp type can't be set */
        if (!type || !br_opt_get(br, BROPT_MULTICAST_STATS_ENABLED))
                return;

        if (p)
                stats = p->mcast_stats;
        else
                stats = br->mcast_stats;
        if (WARN_ON(!stats))
                return;

        br_mcast_stats_add(stats, skb, type, dir);
}

int br_multicast_init_stats(struct net_bridge *br)
{
        br->mcast_stats = netdev_alloc_pcpu_stats(struct bridge_mcast_stats);
        if (!br->mcast_stats)
                return -ENOMEM;

        return 0;
}

void br_multicast_uninit_stats(struct net_bridge *br)
{
        free_percpu(br->mcast_stats);
}

/* noinline for https://llvm.org/pr45802#c9 */
static noinline_for_stack void mcast_stats_add_dir(u64 *dst, u64 *src)
{
        dst[BR_MCAST_DIR_RX] += src[BR_MCAST_DIR_RX];
        dst[BR_MCAST_DIR_TX] += src[BR_MCAST_DIR_TX];
}

void br_multicast_get_stats(const struct net_bridge *br,
                            const struct net_bridge_port *p,
                            struct br_mcast_stats *dest)
{
        struct bridge_mcast_stats __percpu *stats;
        struct br_mcast_stats tdst;
        int i;

        memset(dest, 0, sizeof(*dest));
        if (p)
                stats = p->mcast_stats;
        else
                stats = br->mcast_stats;
        if (WARN_ON(!stats))
                return;

        memset(&tdst, 0, sizeof(tdst));
        for_each_possible_cpu(i) {
                struct bridge_mcast_stats *cpu_stats = per_cpu_ptr(stats, i);
                struct br_mcast_stats temp;
                unsigned int start;

                do {
                        start = u64_stats_fetch_begin(&cpu_stats->syncp);
                        memcpy(&temp, &cpu_stats->mstats, sizeof(temp));
                } while (u64_stats_fetch_retry(&cpu_stats->syncp, start));

                mcast_stats_add_dir(tdst.igmp_v1queries, temp.igmp_v1queries);
                mcast_stats_add_dir(tdst.igmp_v2queries, temp.igmp_v2queries);
                mcast_stats_add_dir(tdst.igmp_v3queries, temp.igmp_v3queries);
                mcast_stats_add_dir(tdst.igmp_leaves, temp.igmp_leaves);
                mcast_stats_add_dir(tdst.igmp_v1reports, temp.igmp_v1reports);
                mcast_stats_add_dir(tdst.igmp_v2reports, temp.igmp_v2reports);
                mcast_stats_add_dir(tdst.igmp_v3reports, temp.igmp_v3reports);
                tdst.igmp_parse_errors += temp.igmp_parse_errors;

                mcast_stats_add_dir(tdst.mld_v1queries, temp.mld_v1queries);
                mcast_stats_add_dir(tdst.mld_v2queries, temp.mld_v2queries);
                mcast_stats_add_dir(tdst.mld_leaves, temp.mld_leaves);
                mcast_stats_add_dir(tdst.mld_v1reports, temp.mld_v1reports);
                mcast_stats_add_dir(tdst.mld_v2reports, temp.mld_v2reports);
                tdst.mld_parse_errors += temp.mld_parse_errors;
        }
        memcpy(dest, &tdst, sizeof(*dest));
}

int br_mdb_hash_init(struct net_bridge *br)
{
        int err;

        err = rhashtable_init(&br->sg_port_tbl, &br_sg_port_rht_params);
        if (err)
                return err;

        err = rhashtable_init(&br->mdb_hash_tbl, &br_mdb_rht_params);
        if (err) {
                rhashtable_destroy(&br->sg_port_tbl);
                return err;
        }

        return 0;
}

void br_mdb_hash_fini(struct net_bridge *br)
{
        rhashtable_destroy(&br->sg_port_tbl);
        rhashtable_destroy(&br->mdb_hash_tbl);
}




































 56446 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
// SPDX-License-Identifier: GPL-2.0-only
#include <linux/fault-inject.h>
#include <linux/fault-inject-usercopy.h>

static struct {
        struct fault_attr attr;
} fail_usercopy = {
        .attr = FAULT_ATTR_INITIALIZER,
};

static int __init setup_fail_usercopy(char *str)
{
        return setup_fault_attr(&fail_usercopy.attr, str);
}
__setup("fail_usercopy=", setup_fail_usercopy);

#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS

static int __init fail_usercopy_debugfs(void)
{
        struct dentry *dir;

        dir = fault_create_debugfs_attr("fail_usercopy", NULL,
                                        &fail_usercopy.attr);
        if (IS_ERR(dir))
                return PTR_ERR(dir);

        return 0;
}

late_initcall(fail_usercopy_debugfs);

#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */

bool should_fail_usercopy(void)
{
        return should_fail(&fail_usercopy.attr, 1);
}
EXPORT_SYMBOL_GPL(should_fail_usercopy);













































    3 























































    3 





























    3 

























































































    3 







































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM compaction

#if !defined(_TRACE_COMPACTION_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_COMPACTION_H

#include <linux/types.h>
#include <linux/list.h>
#include <linux/tracepoint.h>
#include <trace/events/mmflags.h>


DECLARE_EVENT_CLASS(mm_compaction_isolate_template,

        TP_PROTO(
                unsigned long start_pfn,
                unsigned long end_pfn,
                unsigned long nr_scanned,
                unsigned long nr_taken),

        TP_ARGS(start_pfn, end_pfn, nr_scanned, nr_taken),

        TP_STRUCT__entry(
                __field(unsigned long, start_pfn)
                __field(unsigned long, end_pfn)
                __field(unsigned long, nr_scanned)
                __field(unsigned long, nr_taken)
        ),

        TP_fast_assign(
                __entry->start_pfn = start_pfn;
                __entry->end_pfn = end_pfn;
                __entry->nr_scanned = nr_scanned;
                __entry->nr_taken = nr_taken;
        ),

        TP_printk("range=(0x%lx ~ 0x%lx) nr_scanned=%lu nr_taken=%lu",
                __entry->start_pfn,
                __entry->end_pfn,
                __entry->nr_scanned,
                __entry->nr_taken)
);

DEFINE_EVENT(mm_compaction_isolate_template, mm_compaction_isolate_migratepages,

        TP_PROTO(
                unsigned long start_pfn,
                unsigned long end_pfn,
                unsigned long nr_scanned,
                unsigned long nr_taken),

        TP_ARGS(start_pfn, end_pfn, nr_scanned, nr_taken)
);

DEFINE_EVENT(mm_compaction_isolate_template, mm_compaction_isolate_freepages,

        TP_PROTO(
                unsigned long start_pfn,
                unsigned long end_pfn,
                unsigned long nr_scanned,
                unsigned long nr_taken),

        TP_ARGS(start_pfn, end_pfn, nr_scanned, nr_taken)
);

DEFINE_EVENT(mm_compaction_isolate_template, mm_compaction_fast_isolate_freepages,

        TP_PROTO(
                unsigned long start_pfn,
                unsigned long end_pfn,
                unsigned long nr_scanned,
                unsigned long nr_taken),

        TP_ARGS(start_pfn, end_pfn, nr_scanned, nr_taken)
);

#ifdef CONFIG_COMPACTION
TRACE_EVENT(mm_compaction_migratepages,

        TP_PROTO(unsigned int nr_migratepages,
                unsigned int nr_succeeded),

        TP_ARGS(nr_migratepages, nr_succeeded),

        TP_STRUCT__entry(
                __field(unsigned long, nr_migrated)
                __field(unsigned long, nr_failed)
        ),

        TP_fast_assign(
                __entry->nr_migrated = nr_succeeded;
                __entry->nr_failed = nr_migratepages - nr_succeeded;
        ),

        TP_printk("nr_migrated=%lu nr_failed=%lu",
                __entry->nr_migrated,
                __entry->nr_failed)
);

TRACE_EVENT(mm_compaction_begin,
        TP_PROTO(struct compact_control *cc, unsigned long zone_start,
                unsigned long zone_end, bool sync),

        TP_ARGS(cc, zone_start, zone_end, sync),

        TP_STRUCT__entry(
                __field(unsigned long, zone_start)
                __field(unsigned long, migrate_pfn)
                __field(unsigned long, free_pfn)
                __field(unsigned long, zone_end)
                __field(bool, sync)
        ),

        TP_fast_assign(
                __entry->zone_start = zone_start;
                __entry->migrate_pfn = cc->migrate_pfn;
                __entry->free_pfn = cc->free_pfn;
                __entry->zone_end = zone_end;
                __entry->sync = sync;
        ),

        TP_printk("zone_start=0x%lx migrate_pfn=0x%lx free_pfn=0x%lx zone_end=0x%lx, mode=%s",
                __entry->zone_start,
                __entry->migrate_pfn,
                __entry->free_pfn,
                __entry->zone_end,
                __entry->sync ? "sync" : "async")
);

TRACE_EVENT(mm_compaction_end,
        TP_PROTO(struct compact_control *cc, unsigned long zone_start,
                unsigned long zone_end, bool sync,
                int status),

        TP_ARGS(cc, zone_start, zone_end, sync, status),

        TP_STRUCT__entry(
                __field(unsigned long, zone_start)
                __field(unsigned long, migrate_pfn)
                __field(unsigned long, free_pfn)
                __field(unsigned long, zone_end)
                __field(bool, sync)
                __field(int, status)
        ),

        TP_fast_assign(
                __entry->zone_start = zone_start;
                __entry->migrate_pfn = cc->migrate_pfn;
                __entry->free_pfn = cc->free_pfn;
                __entry->zone_end = zone_end;
                __entry->sync = sync;
                __entry->status = status;
        ),

        TP_printk("zone_start=0x%lx migrate_pfn=0x%lx free_pfn=0x%lx zone_end=0x%lx, mode=%s status=%s",
                __entry->zone_start,
                __entry->migrate_pfn,
                __entry->free_pfn,
                __entry->zone_end,
                __entry->sync ? "sync" : "async",
                __print_symbolic(__entry->status, COMPACTION_STATUS))
);

TRACE_EVENT(mm_compaction_try_to_compact_pages,

        TP_PROTO(
                int order,
                gfp_t gfp_mask,
                int prio),

        TP_ARGS(order, gfp_mask, prio),

        TP_STRUCT__entry(
                __field(int, order)
                __field(unsigned long, gfp_mask)
                __field(int, prio)
        ),

        TP_fast_assign(
                __entry->order = order;
                __entry->gfp_mask = (__force unsigned long)gfp_mask;
                __entry->prio = prio;
        ),

        TP_printk("order=%d gfp_mask=%s priority=%d",
                __entry->order,
                show_gfp_flags(__entry->gfp_mask),
                __entry->prio)
);

DECLARE_EVENT_CLASS(mm_compaction_suitable_template,

        TP_PROTO(struct zone *zone,
                int order,
                int ret),

        TP_ARGS(zone, order, ret),

        TP_STRUCT__entry(
                __field(int, nid)
                __field(enum zone_type, idx)
                __field(int, order)
                __field(int, ret)
        ),

        TP_fast_assign(
                __entry->nid = zone_to_nid(zone);
                __entry->idx = zone_idx(zone);
                __entry->order = order;
                __entry->ret = ret;
        ),

        TP_printk("node=%d zone=%-8s order=%d ret=%s",
                __entry->nid,
                __print_symbolic(__entry->idx, ZONE_TYPE),
                __entry->order,
                __print_symbolic(__entry->ret, COMPACTION_STATUS))
);

DEFINE_EVENT(mm_compaction_suitable_template, mm_compaction_finished,

        TP_PROTO(struct zone *zone,
                int order,
                int ret),

        TP_ARGS(zone, order, ret)
);

DEFINE_EVENT(mm_compaction_suitable_template, mm_compaction_suitable,

        TP_PROTO(struct zone *zone,
                int order,
                int ret),

        TP_ARGS(zone, order, ret)
);

DECLARE_EVENT_CLASS(mm_compaction_defer_template,

        TP_PROTO(struct zone *zone, int order),

        TP_ARGS(zone, order),

        TP_STRUCT__entry(
                __field(int, nid)
                __field(enum zone_type, idx)
                __field(int, order)
                __field(unsigned int, considered)
                __field(unsigned int, defer_shift)
                __field(int, order_failed)
        ),

        TP_fast_assign(
                __entry->nid = zone_to_nid(zone);
                __entry->idx = zone_idx(zone);
                __entry->order = order;
                __entry->considered = zone->compact_considered;
                __entry->defer_shift = zone->compact_defer_shift;
                __entry->order_failed = zone->compact_order_failed;
        ),

        TP_printk("node=%d zone=%-8s order=%d order_failed=%d consider=%u limit=%lu",
                __entry->nid,
                __print_symbolic(__entry->idx, ZONE_TYPE),
                __entry->order,
                __entry->order_failed,
                __entry->considered,
                1UL << __entry->defer_shift)
);

DEFINE_EVENT(mm_compaction_defer_template, mm_compaction_deferred,

        TP_PROTO(struct zone *zone, int order),

        TP_ARGS(zone, order)
);

DEFINE_EVENT(mm_compaction_defer_template, mm_compaction_defer_compaction,

        TP_PROTO(struct zone *zone, int order),

        TP_ARGS(zone, order)
);

DEFINE_EVENT(mm_compaction_defer_template, mm_compaction_defer_reset,

        TP_PROTO(struct zone *zone, int order),

        TP_ARGS(zone, order)
);

TRACE_EVENT(mm_compaction_kcompactd_sleep,

        TP_PROTO(int nid),

        TP_ARGS(nid),

        TP_STRUCT__entry(
                __field(int, nid)
        ),

        TP_fast_assign(
                __entry->nid = nid;
        ),

        TP_printk("nid=%d", __entry->nid)
);

DECLARE_EVENT_CLASS(kcompactd_wake_template,

        TP_PROTO(int nid, int order, enum zone_type highest_zoneidx),

        TP_ARGS(nid, order, highest_zoneidx),

        TP_STRUCT__entry(
                __field(int, nid)
                __field(int, order)
                __field(enum zone_type, highest_zoneidx)
        ),

        TP_fast_assign(
                __entry->nid = nid;
                __entry->order = order;
                __entry->highest_zoneidx = highest_zoneidx;
        ),

        /*
         * classzone_idx is previous name of the highest_zoneidx.
         * Reason not to change it is the ABI requirement of the tracepoint.
         */
        TP_printk("nid=%d order=%d classzone_idx=%-8s",
                __entry->nid,
                __entry->order,
                __print_symbolic(__entry->highest_zoneidx, ZONE_TYPE))
);

DEFINE_EVENT(kcompactd_wake_template, mm_compaction_wakeup_kcompactd,

        TP_PROTO(int nid, int order, enum zone_type highest_zoneidx),

        TP_ARGS(nid, order, highest_zoneidx)
);

DEFINE_EVENT(kcompactd_wake_template, mm_compaction_kcompactd_wake,

        TP_PROTO(int nid, int order, enum zone_type highest_zoneidx),

        TP_ARGS(nid, order, highest_zoneidx)
);
#endif

#endif /* _TRACE_COMPACTION_H */

/* This part must be outside protection */
#include <trace/define_trace.h>







































































































    2 




    1 









1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef LINUX_MLD_H
#define LINUX_MLD_H

#include <linux/in6.h>
#include <linux/icmpv6.h>

/* MLDv1 Query/Report/Done */
struct mld_msg {
        struct icmp6hdr                mld_hdr;
        struct in6_addr                mld_mca;
};

#define mld_type                mld_hdr.icmp6_type
#define mld_code                mld_hdr.icmp6_code
#define mld_cksum                mld_hdr.icmp6_cksum
#define mld_maxdelay                mld_hdr.icmp6_maxdelay
#define mld_reserved                mld_hdr.icmp6_dataun.un_data16[1]

/* Multicast Listener Discovery version 2 headers */
/* MLDv2 Report */
struct mld2_grec {
        __u8                grec_type;
        __u8                grec_auxwords;
        __be16                grec_nsrcs;
        struct in6_addr        grec_mca;
        struct in6_addr        grec_src[];
};

struct mld2_report {
        struct icmp6hdr                mld2r_hdr;
        struct mld2_grec        mld2r_grec[];
};

#define mld2r_type                mld2r_hdr.icmp6_type
#define mld2r_resv1                mld2r_hdr.icmp6_code
#define mld2r_cksum                mld2r_hdr.icmp6_cksum
#define mld2r_resv2                mld2r_hdr.icmp6_dataun.un_data16[0]
#define mld2r_ngrec                mld2r_hdr.icmp6_dataun.un_data16[1]

/* MLDv2 Query */
struct mld2_query {
        struct icmp6hdr                mld2q_hdr;
        struct in6_addr                mld2q_mca;
#if defined(__LITTLE_ENDIAN_BITFIELD)
        __u8                        mld2q_qrv:3,
                                mld2q_suppress:1,
                                mld2q_resv2:4;
#elif defined(__BIG_ENDIAN_BITFIELD)
        __u8                        mld2q_resv2:4,
                                mld2q_suppress:1,
                                mld2q_qrv:3;
#else
#error "Please fix <asm/byteorder.h>"
#endif
        __u8                        mld2q_qqic;
        __be16                        mld2q_nsrcs;
        struct in6_addr                mld2q_srcs[];
};

#define mld2q_type                mld2q_hdr.icmp6_type
#define mld2q_code                mld2q_hdr.icmp6_code
#define mld2q_cksum                mld2q_hdr.icmp6_cksum
#define mld2q_mrc                mld2q_hdr.icmp6_maxdelay
#define mld2q_resv1                mld2q_hdr.icmp6_dataun.un_data16[1]

/* RFC3810, 5.1.3. Maximum Response Code:
 *
 * If Maximum Response Code >= 32768, Maximum Response Code represents a
 * floating-point value as follows:
 *
 *  0 1 2 3 4 5 6 7 8 9 A B C D E F
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 * |1| exp |          mant         |
 * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 */
#define MLDV2_MRC_EXP(value)        (((value) >> 12) & 0x0007)
#define MLDV2_MRC_MAN(value)        ((value) & 0x0fff)

/* RFC3810, 5.1.9. QQIC (Querier's Query Interval Code):
 *
 * If QQIC >= 128, QQIC represents a floating-point value as follows:
 *
 *  0 1 2 3 4 5 6 7
 * +-+-+-+-+-+-+-+-+
 * |1| exp | mant  |
 * +-+-+-+-+-+-+-+-+
 */
#define MLDV2_QQIC_EXP(value)        (((value) >> 4) & 0x07)
#define MLDV2_QQIC_MAN(value)        ((value) & 0x0f)

#define MLD_EXP_MIN_LIMIT        32768UL
#define MLDV1_MRD_MAX_COMPAT        (MLD_EXP_MIN_LIMIT - 1)

#define MLD_MAX_QUEUE                8
#define MLD_MAX_SKBS                32

static inline unsigned long mldv2_mrc(const struct mld2_query *mlh2)
{
        /* RFC3810, 5.1.3. Maximum Response Code */
        unsigned long ret, mc_mrc = ntohs(mlh2->mld2q_mrc);

        if (mc_mrc < MLD_EXP_MIN_LIMIT) {
                ret = mc_mrc;
        } else {
                unsigned long mc_man, mc_exp;

                mc_exp = MLDV2_MRC_EXP(mc_mrc);
                mc_man = MLDV2_MRC_MAN(mc_mrc);

                ret = (mc_man | 0x1000) << (mc_exp + 3);
        }

        return ret;
}

#endif



























































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    1 





    1 




    1 








    1 


























    1 


















    1 








































































































    1 








































    1 




    1 




    1 
    1 



    3 
    3 









    3 
    3 
    3 





    2 





    2 


























































































































    1 






    1 





























































































    1 






    1 












































































    1 







































    1 

























    1 




    1 
































































    1 

















    1 


    1 





    1 




















    1 
    1 









    1 




















    9 






















    8 





    9 






































    9 







    9 




















    9 



































    9 















    1 




















    1 
















































    1 


    1 














    1 






















    1 


















   42 














   42 











































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Syscall interface to knfsd.
 *
 * Copyright (C) 1995, 1996 Olaf Kirch <okir@monad.swb.de>
 */

#include <linux/slab.h>
#include <linux/namei.h>
#include <linux/ctype.h>
#include <linux/fs_context.h>

#include <linux/sunrpc/svcsock.h>
#include <linux/lockd/lockd.h>
#include <linux/sunrpc/addr.h>
#include <linux/sunrpc/gss_api.h>
#include <linux/sunrpc/rpc_pipe_fs.h>
#include <linux/sunrpc/svc.h>
#include <linux/module.h>
#include <linux/fsnotify.h>
#include <linux/nfslocalio.h>

#include "idmap.h"
#include "nfsd.h"
#include "cache.h"
#include "state.h"
#include "netns.h"
#include "pnfs.h"
#include "filecache.h"
#include "trace.h"
#include "netlink.h"

/*
 *        We have a single directory with several nodes in it.
 */
enum {
        NFSD_Root = 1,
        NFSD_List,
        NFSD_Export_Stats,
        NFSD_Export_features,
        NFSD_Fh,
        NFSD_FO_UnlockIP,
        NFSD_FO_UnlockFS,
        NFSD_Threads,
        NFSD_Pool_Threads,
        NFSD_Pool_Stats,
        NFSD_Reply_Cache_Stats,
        NFSD_Versions,
        NFSD_Ports,
        NFSD_MaxBlkSize,
        NFSD_MaxConnections,
        NFSD_Filecache,
        NFSD_Leasetime,
        NFSD_Gracetime,
        NFSD_RecoveryDir,
        NFSD_V4EndGrace,
        NFSD_MaxReserved
};

/*
 * write() for these nodes.
 */
static ssize_t write_filehandle(struct file *file, char *buf, size_t size);
static ssize_t write_unlock_ip(struct file *file, char *buf, size_t size);
static ssize_t write_unlock_fs(struct file *file, char *buf, size_t size);
static ssize_t write_threads(struct file *file, char *buf, size_t size);
static ssize_t write_pool_threads(struct file *file, char *buf, size_t size);
static ssize_t write_versions(struct file *file, char *buf, size_t size);
static ssize_t write_ports(struct file *file, char *buf, size_t size);
static ssize_t write_maxblksize(struct file *file, char *buf, size_t size);
static ssize_t write_maxconn(struct file *file, char *buf, size_t size);
#ifdef CONFIG_NFSD_V4
static ssize_t write_leasetime(struct file *file, char *buf, size_t size);
static ssize_t write_gracetime(struct file *file, char *buf, size_t size);
#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
static ssize_t write_recoverydir(struct file *file, char *buf, size_t size);
#endif
static ssize_t write_v4_end_grace(struct file *file, char *buf, size_t size);
#endif

static ssize_t (*const write_op[])(struct file *, char *, size_t) = {
        [NFSD_Fh] = write_filehandle,
        [NFSD_FO_UnlockIP] = write_unlock_ip,
        [NFSD_FO_UnlockFS] = write_unlock_fs,
        [NFSD_Threads] = write_threads,
        [NFSD_Pool_Threads] = write_pool_threads,
        [NFSD_Versions] = write_versions,
        [NFSD_Ports] = write_ports,
        [NFSD_MaxBlkSize] = write_maxblksize,
        [NFSD_MaxConnections] = write_maxconn,
#ifdef CONFIG_NFSD_V4
        [NFSD_Leasetime] = write_leasetime,
        [NFSD_Gracetime] = write_gracetime,
#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
        [NFSD_RecoveryDir] = write_recoverydir,
#endif
        [NFSD_V4EndGrace] = write_v4_end_grace,
#endif
};

static ssize_t nfsctl_transaction_write(struct file *file, const char __user *buf, size_t size, loff_t *pos)
{
        ino_t ino =  file_inode(file)->i_ino;
        char *data;
        ssize_t rv;

        if (ino >= ARRAY_SIZE(write_op) || !write_op[ino])
                return -EINVAL;

        data = simple_transaction_get(file, buf, size);
        if (IS_ERR(data))
                return PTR_ERR(data);

        rv = write_op[ino](file, data, size);
        if (rv < 0)
                return rv;

        simple_transaction_set(file, rv);
        return size;
}

static ssize_t nfsctl_transaction_read(struct file *file, char __user *buf, size_t size, loff_t *pos)
{
        if (! file->private_data) {
                /* An attempt to read a transaction file without writing
                 * causes a 0-byte write so that the file can return
                 * state information
                 */
                ssize_t rv = nfsctl_transaction_write(file, buf, 0, pos);
                if (rv < 0)
                        return rv;
        }
        return simple_transaction_read(file, buf, size, pos);
}

static const struct file_operations transaction_ops = {
        .write                = nfsctl_transaction_write,
        .read                = nfsctl_transaction_read,
        .release        = simple_transaction_release,
        .llseek                = default_llseek,
};

static int exports_net_open(struct net *net, struct file *file)
{
        int err;
        struct seq_file *seq;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        err = seq_open(file, &nfs_exports_op);
        if (err)
                return err;

        seq = file->private_data;
        seq->private = nn->svc_export_cache;
        return 0;
}

static int exports_nfsd_open(struct inode *inode, struct file *file)
{
        return exports_net_open(inode->i_sb->s_fs_info, file);
}

static const struct file_operations exports_nfsd_operations = {
        .open                = exports_nfsd_open,
        .read                = seq_read,
        .llseek                = seq_lseek,
        .release        = seq_release,
};

static int export_features_show(struct seq_file *m, void *v)
{
        seq_printf(m, "0x%x 0x%x\n", NFSEXP_ALLFLAGS, NFSEXP_SECINFO_FLAGS);
        return 0;
}

DEFINE_SHOW_ATTRIBUTE(export_features);

static int nfsd_pool_stats_open(struct inode *inode, struct file *file)
{
        struct nfsd_net *nn = net_generic(inode->i_sb->s_fs_info, nfsd_net_id);

        return svc_pool_stats_open(&nn->nfsd_info, file);
}

static const struct file_operations pool_stats_operations = {
        .open                = nfsd_pool_stats_open,
        .read                = seq_read,
        .llseek                = seq_lseek,
        .release        = seq_release,
};

DEFINE_SHOW_ATTRIBUTE(nfsd_reply_cache_stats);

DEFINE_SHOW_ATTRIBUTE(nfsd_file_cache_stats);

/*----------------------------------------------------------------------------*/
/*
 * payload - write methods
 */

static inline struct net *netns(struct file *file)
{
        return file_inode(file)->i_sb->s_fs_info;
}

/*
 * write_unlock_ip - Release all locks used by a client
 *
 * Experimental.
 *
 * Input:
 *                        buf:        '\n'-terminated C string containing a
 *                                presentation format IP address
 *                        size:        length of C string in @buf
 * Output:
 *        On success:        returns zero if all specified locks were released;
 *                        returns one if one or more locks were not released
 *        On error:        return code is negative errno value
 */
static ssize_t write_unlock_ip(struct file *file, char *buf, size_t size)
{
        struct sockaddr_storage address;
        struct sockaddr *sap = (struct sockaddr *)&address;
        size_t salen = sizeof(address);
        char *fo_path;
        struct net *net = netns(file);

        /* sanity check */
        if (size == 0)
                return -EINVAL;

        if (buf[size-1] != '\n')
                return -EINVAL;

        fo_path = buf;
        if (qword_get(&buf, fo_path, size) < 0)
                return -EINVAL;

        if (rpc_pton(net, fo_path, size, sap, salen) == 0)
                return -EINVAL;

        trace_nfsd_ctl_unlock_ip(net, buf);
        return nlmsvc_unlock_all_by_ip(sap);
}

/*
 * write_unlock_fs - Release all locks on a local file system
 *
 * Experimental.
 *
 * Input:
 *                        buf:        '\n'-terminated C string containing the
 *                                absolute pathname of a local file system
 *                        size:        length of C string in @buf
 * Output:
 *        On success:        returns zero if all specified locks were released;
 *                        returns one if one or more locks were not released
 *        On error:        return code is negative errno value
 */
static ssize_t write_unlock_fs(struct file *file, char *buf, size_t size)
{
        struct path path;
        char *fo_path;
        int error;

        /* sanity check */
        if (size == 0)
                return -EINVAL;

        if (buf[size-1] != '\n')
                return -EINVAL;

        fo_path = buf;
        if (qword_get(&buf, fo_path, size) < 0)
                return -EINVAL;
        trace_nfsd_ctl_unlock_fs(netns(file), fo_path);
        error = kern_path(fo_path, 0, &path);
        if (error)
                return error;

        /*
         * XXX: Needs better sanity checking.  Otherwise we could end up
         * releasing locks on the wrong file system.
         *
         * For example:
         * 1.  Does the path refer to a directory?
         * 2.  Is that directory a mount point, or
         * 3.  Is that directory the root of an exported file system?
         */
        error = nlmsvc_unlock_all_by_sb(path.dentry->d_sb);
        nfsd4_revoke_states(netns(file), path.dentry->d_sb);

        path_put(&path);
        return error;
}

/*
 * write_filehandle - Get a variable-length NFS file handle by path
 *
 * On input, the buffer contains a '\n'-terminated C string comprised of
 * three alphanumeric words separated by whitespace.  The string may
 * contain escape sequences.
 *
 * Input:
 *                        buf:
 *                                domain:                client domain name
 *                                path:                export pathname
 *                                maxsize:        numeric maximum size of
 *                                                @buf
 *                        size:        length of C string in @buf
 * Output:
 *        On success:        passed-in buffer filled with '\n'-terminated C
 *                        string containing a ASCII hex text version
 *                        of the NFS file handle;
 *                        return code is the size in bytes of the string
 *        On error:        return code is negative errno value
 */
static ssize_t write_filehandle(struct file *file, char *buf, size_t size)
{
        char *dname, *path;
        int maxsize;
        char *mesg = buf;
        int len;
        struct auth_domain *dom;
        struct knfsd_fh fh;

        if (size == 0)
                return -EINVAL;

        if (buf[size-1] != '\n')
                return -EINVAL;
        buf[size-1] = 0;

        dname = mesg;
        len = qword_get(&mesg, dname, size);
        if (len <= 0)
                return -EINVAL;

        path = dname+len+1;
        len = qword_get(&mesg, path, size);
        if (len <= 0)
                return -EINVAL;

        len = get_int(&mesg, &maxsize);
        if (len)
                return len;

        if (maxsize < NFS_FHSIZE)
                return -EINVAL;
        maxsize = min(maxsize, NFS3_FHSIZE);

        if (qword_get(&mesg, mesg, size) > 0)
                return -EINVAL;

        trace_nfsd_ctl_filehandle(netns(file), dname, path, maxsize);

        /* we have all the words, they are in buf.. */
        dom = unix_domain_find(dname);
        if (!dom)
                return -ENOMEM;

        len = exp_rootfh(netns(file), dom, path, &fh, maxsize);
        auth_domain_put(dom);
        if (len)
                return len;

        mesg = buf;
        len = SIMPLE_TRANSACTION_LIMIT;
        qword_addhex(&mesg, &len, fh.fh_raw, fh.fh_size);
        mesg[-1] = '\n';
        return mesg - buf;
}

/*
 * write_threads - Start NFSD, or report the current number of running threads
 *
 * Input:
 *                        buf:                ignored
 *                        size:                zero
 * Output:
 *        On success:        passed-in buffer filled with '\n'-terminated C
 *                        string numeric value representing the number of
 *                        running NFSD threads;
 *                        return code is the size in bytes of the string
 *        On error:        return code is zero
 *
 * OR
 *
 * Input:
 *                        buf:                C string containing an unsigned
 *                                        integer value representing the
 *                                        number of NFSD threads to start
 *                        size:                non-zero length of C string in @buf
 * Output:
 *        On success:        NFS service is started;
 *                        passed-in buffer filled with '\n'-terminated C
 *                        string numeric value representing the number of
 *                        running NFSD threads;
 *                        return code is the size in bytes of the string
 *        On error:        return code is zero or a negative errno value
 */
static ssize_t write_threads(struct file *file, char *buf, size_t size)
{
        char *mesg = buf;
        int rv;
        struct net *net = netns(file);

        if (size > 0) {
                int newthreads;
                rv = get_int(&mesg, &newthreads);
                if (rv)
                        return rv;
                if (newthreads < 0)
                        return -EINVAL;
                trace_nfsd_ctl_threads(net, newthreads);
                mutex_lock(&nfsd_mutex);
                rv = nfsd_svc(1, &newthreads, net, file->f_cred, NULL);
                mutex_unlock(&nfsd_mutex);
                if (rv < 0)
                        return rv;
        } else
                rv = nfsd_nrthreads(net);

        return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%d\n", rv);
}

/*
 * write_pool_threads - Set or report the current number of threads per pool
 *
 * Input:
 *                        buf:                ignored
 *                        size:                zero
 *
 * OR
 *
 * Input:
 *                        buf:                C string containing whitespace-
 *                                        separated unsigned integer values
 *                                        representing the number of NFSD
 *                                        threads to start in each pool
 *                        size:                non-zero length of C string in @buf
 * Output:
 *        On success:        passed-in buffer filled with '\n'-terminated C
 *                        string containing integer values representing the
 *                        number of NFSD threads in each pool;
 *                        return code is the size in bytes of the string
 *        On error:        return code is zero or a negative errno value
 */
static ssize_t write_pool_threads(struct file *file, char *buf, size_t size)
{
        /* if size > 0, look for an array of number of threads per node
         * and apply them  then write out number of threads per node as reply
         */
        char *mesg = buf;
        int i;
        int rv;
        int len;
        int npools;
        int *nthreads;
        struct net *net = netns(file);

        mutex_lock(&nfsd_mutex);
        npools = nfsd_nrpools(net);
        if (npools == 0) {
                /*
                 * NFS is shut down.  The admin can start it by
                 * writing to the threads file but NOT the pool_threads
                 * file, sorry.  Report zero threads.
                 */
                mutex_unlock(&nfsd_mutex);
                strcpy(buf, "0\n");
                return strlen(buf);
        }

        nthreads = kcalloc(npools, sizeof(int), GFP_KERNEL);
        rv = -ENOMEM;
        if (nthreads == NULL)
                goto out_free;

        if (size > 0) {
                for (i = 0; i < npools; i++) {
                        rv = get_int(&mesg, &nthreads[i]);
                        if (rv == -ENOENT)
                                break;                /* fewer numbers than pools */
                        if (rv)
                                goto out_free;        /* syntax error */
                        rv = -EINVAL;
                        if (nthreads[i] < 0)
                                goto out_free;
                        trace_nfsd_ctl_pool_threads(net, i, nthreads[i]);
                }

                /*
                 * There must always be a thread in pool 0; the admin
                 * can't shut down NFS completely using pool_threads.
                 */
                if (nthreads[0] == 0)
                        nthreads[0] = 1;

                rv = nfsd_set_nrthreads(i, nthreads, net);
                if (rv)
                        goto out_free;
        }

        rv = nfsd_get_nrthreads(npools, nthreads, net);
        if (rv)
                goto out_free;

        mesg = buf;
        size = SIMPLE_TRANSACTION_LIMIT;
        for (i = 0; i < npools && size > 0; i++) {
                snprintf(mesg, size, "%d%c", nthreads[i], (i == npools-1 ? '\n' : ' '));
                len = strlen(mesg);
                size -= len;
                mesg += len;
        }
        rv = mesg - buf;
out_free:
        kfree(nthreads);
        mutex_unlock(&nfsd_mutex);
        return rv;
}

static ssize_t
nfsd_print_version_support(struct nfsd_net *nn, char *buf, int remaining,
                const char *sep, unsigned vers, int minor)
{
        const char *format = minor < 0 ? "%s%c%u" : "%s%c%u.%u";
        bool supported = !!nfsd_vers(nn, vers, NFSD_TEST);

        if (vers == 4 && minor >= 0 &&
            !nfsd_minorversion(nn, minor, NFSD_TEST))
                supported = false;
        if (minor == 0 && supported)
                /*
                 * special case for backward compatability.
                 * +4.0 is never reported, it is implied by
                 * +4, unless -4.0 is present.
                 */
                return 0;
        return snprintf(buf, remaining, format, sep,
                        supported ? '+' : '-', vers, minor);
}

static ssize_t __write_versions(struct file *file, char *buf, size_t size)
{
        char *mesg = buf;
        char *vers, *minorp, sign;
        int len, num, remaining;
        ssize_t tlen = 0;
        char *sep;
        struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id);

        if (size > 0) {
                if (nn->nfsd_serv)
                        /* Cannot change versions without updating
                         * nn->nfsd_serv->sv_xdrsize, and reallocing
                         * rq_argp and rq_resp
                         */
                        return -EBUSY;
                if (buf[size-1] != '\n')
                        return -EINVAL;
                buf[size-1] = 0;
                trace_nfsd_ctl_version(netns(file), buf);

                vers = mesg;
                len = qword_get(&mesg, vers, size);
                if (len <= 0) return -EINVAL;
                do {
                        enum vers_op cmd;
                        unsigned minor;
                        sign = *vers;
                        if (sign == '+' || sign == '-')
                                num = simple_strtol((vers+1), &minorp, 0);
                        else
                                num = simple_strtol(vers, &minorp, 0);
                        if (*minorp == '.') {
                                if (num != 4)
                                        return -EINVAL;
                                if (kstrtouint(minorp+1, 0, &minor) < 0)
                                        return -EINVAL;
                        }

                        cmd = sign == '-' ? NFSD_CLEAR : NFSD_SET;
                        switch(num) {
#ifdef CONFIG_NFSD_V2
                        case 2:
#endif
                        case 3:
                                nfsd_vers(nn, num, cmd);
                                break;
                        case 4:
                                if (*minorp == '.') {
                                        if (nfsd_minorversion(nn, minor, cmd) < 0)
                                                return -EINVAL;
                                } else if ((cmd == NFSD_SET) != nfsd_vers(nn, num, NFSD_TEST)) {
                                        /*
                                         * Either we have +4 and no minors are enabled,
                                         * or we have -4 and at least one minor is enabled.
                                         * In either case, propagate 'cmd' to all minors.
                                         */
                                        minor = 0;
                                        while (nfsd_minorversion(nn, minor, cmd) >= 0)
                                                minor++;
                                }
                                break;
                        default:
                                /* Ignore requests to disable non-existent versions */
                                if (cmd == NFSD_SET)
                                        return -EINVAL;
                        }
                        vers += len + 1;
                } while ((len = qword_get(&mesg, vers, size)) > 0);
                /* If all get turned off, turn them back on, as
                 * having no versions is BAD
                 */
                nfsd_reset_versions(nn);
        }

        /* Now write current state into reply buffer */
        sep = "";
        remaining = SIMPLE_TRANSACTION_LIMIT;
        for (num=2 ; num <= 4 ; num++) {
                int minor;
                if (!nfsd_vers(nn, num, NFSD_AVAIL))
                        continue;

                minor = -1;
                do {
                        len = nfsd_print_version_support(nn, buf, remaining,
                                        sep, num, minor);
                        if (len >= remaining)
                                goto out;
                        remaining -= len;
                        buf += len;
                        tlen += len;
                        minor++;
                        if (len)
                                sep = " ";
                } while (num == 4 && minor <= NFSD_SUPPORTED_MINOR_VERSION);
        }
out:
        len = snprintf(buf, remaining, "\n");
        if (len >= remaining)
                return -EINVAL;
        return tlen + len;
}

/*
 * write_versions - Set or report the available NFS protocol versions
 *
 * Input:
 *                        buf:                ignored
 *                        size:                zero
 * Output:
 *        On success:        passed-in buffer filled with '\n'-terminated C
 *                        string containing positive or negative integer
 *                        values representing the current status of each
 *                        protocol version;
 *                        return code is the size in bytes of the string
 *        On error:        return code is zero or a negative errno value
 *
 * OR
 *
 * Input:
 *                        buf:                C string containing whitespace-
 *                                        separated positive or negative
 *                                        integer values representing NFS
 *                                        protocol versions to enable ("+n")
 *                                        or disable ("-n")
 *                        size:                non-zero length of C string in @buf
 * Output:
 *        On success:        status of zero or more protocol versions has
 *                        been updated; passed-in buffer filled with
 *                        '\n'-terminated C string containing positive
 *                        or negative integer values representing the
 *                        current status of each protocol version;
 *                        return code is the size in bytes of the string
 *        On error:        return code is zero or a negative errno value
 */
static ssize_t write_versions(struct file *file, char *buf, size_t size)
{
        ssize_t rv;

        mutex_lock(&nfsd_mutex);
        rv = __write_versions(file, buf, size);
        mutex_unlock(&nfsd_mutex);
        return rv;
}

/*
 * Zero-length write.  Return a list of NFSD's current listener
 * transports.
 */
static ssize_t __write_ports_names(char *buf, struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        if (nn->nfsd_serv == NULL)
                return 0;
        return svc_xprt_names(nn->nfsd_serv, buf, SIMPLE_TRANSACTION_LIMIT);
}

/*
 * A single 'fd' number was written, in which case it must be for
 * a socket of a supported family/protocol, and we use it as an
 * nfsd listener.
 */
static ssize_t __write_ports_addfd(char *buf, struct net *net, const struct cred *cred)
{
        char *mesg = buf;
        int fd, err;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv;

        err = get_int(&mesg, &fd);
        if (err != 0 || fd < 0)
                return -EINVAL;
        trace_nfsd_ctl_ports_addfd(net, fd);

        err = nfsd_create_serv(net);
        if (err != 0)
                return err;

        serv = nn->nfsd_serv;
        err = svc_addsock(serv, net, fd, buf, SIMPLE_TRANSACTION_LIMIT, cred);

        if (!serv->sv_nrthreads && list_empty(&nn->nfsd_serv->sv_permsocks))
                nfsd_destroy_serv(net);

        return err;
}

/*
 * A transport listener is added by writing its transport name and
 * a port number.
 */
static ssize_t __write_ports_addxprt(char *buf, struct net *net, const struct cred *cred)
{
        char transport[16];
        struct svc_xprt *xprt;
        int port, err;
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        struct svc_serv *serv;

        if (sscanf(buf, "%15s %5u", transport, &port) != 2)
                return -EINVAL;

        if (port < 1 || port > USHRT_MAX)
                return -EINVAL;
        trace_nfsd_ctl_ports_addxprt(net, transport, port);

        err = nfsd_create_serv(net);
        if (err != 0)
                return err;

        serv = nn->nfsd_serv;
        err = svc_xprt_create(serv, transport, net,
                              PF_INET, port, SVC_SOCK_ANONYMOUS, cred);
        if (err < 0)
                goto out_err;

        err = svc_xprt_create(serv, transport, net,
                              PF_INET6, port, SVC_SOCK_ANONYMOUS, cred);
        if (err < 0 && err != -EAFNOSUPPORT)
                goto out_close;

        return 0;
out_close:
        xprt = svc_find_xprt(serv, transport, net, PF_INET, port);
        if (xprt != NULL) {
                svc_xprt_close(xprt);
                svc_xprt_put(xprt);
        }
out_err:
        if (!serv->sv_nrthreads && list_empty(&nn->nfsd_serv->sv_permsocks))
                nfsd_destroy_serv(net);

        return err;
}

static ssize_t __write_ports(struct file *file, char *buf, size_t size,
                             struct net *net)
{
        if (size == 0)
                return __write_ports_names(buf, net);

        if (isdigit(buf[0]))
                return __write_ports_addfd(buf, net, file->f_cred);

        if (isalpha(buf[0]))
                return __write_ports_addxprt(buf, net, file->f_cred);

        return -EINVAL;
}

/*
 * write_ports - Pass a socket file descriptor or transport name to listen on
 *
 * Input:
 *                        buf:                ignored
 *                        size:                zero
 * Output:
 *        On success:        passed-in buffer filled with a '\n'-terminated C
 *                        string containing a whitespace-separated list of
 *                        named NFSD listeners;
 *                        return code is the size in bytes of the string
 *        On error:        return code is zero or a negative errno value
 *
 * OR
 *
 * Input:
 *                        buf:                C string containing an unsigned
 *                                        integer value representing a bound
 *                                        but unconnected socket that is to be
 *                                        used as an NFSD listener; listen(3)
 *                                        must be called for a SOCK_STREAM
 *                                        socket, otherwise it is ignored
 *                        size:                non-zero length of C string in @buf
 * Output:
 *        On success:        NFS service is started;
 *                        passed-in buffer filled with a '\n'-terminated C
 *                        string containing a unique alphanumeric name of
 *                        the listener;
 *                        return code is the size in bytes of the string
 *        On error:        return code is a negative errno value
 *
 * OR
 *
 * Input:
 *                        buf:                C string containing a transport
 *                                        name and an unsigned integer value
 *                                        representing the port to listen on,
 *                                        separated by whitespace
 *                        size:                non-zero length of C string in @buf
 * Output:
 *        On success:        returns zero; NFS service is started
 *        On error:        return code is a negative errno value
 */
static ssize_t write_ports(struct file *file, char *buf, size_t size)
{
        ssize_t rv;

        mutex_lock(&nfsd_mutex);
        rv = __write_ports(file, buf, size, netns(file));
        mutex_unlock(&nfsd_mutex);
        return rv;
}


int nfsd_max_blksize;

/*
 * write_maxblksize - Set or report the current NFS blksize
 *
 * Input:
 *                        buf:                ignored
 *                        size:                zero
 *
 * OR
 *
 * Input:
 *                        buf:                C string containing an unsigned
 *                                        integer value representing the new
 *                                        NFS blksize
 *                        size:                non-zero length of C string in @buf
 * Output:
 *        On success:        passed-in buffer filled with '\n'-terminated C string
 *                        containing numeric value of the current NFS blksize
 *                        setting;
 *                        return code is the size in bytes of the string
 *        On error:        return code is zero or a negative errno value
 */
static ssize_t write_maxblksize(struct file *file, char *buf, size_t size)
{
        char *mesg = buf;
        struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id);

        if (size > 0) {
                int bsize;
                int rv = get_int(&mesg, &bsize);
                if (rv)
                        return rv;
                trace_nfsd_ctl_maxblksize(netns(file), bsize);

                /* force bsize into allowed range and
                 * required alignment.
                 */
                bsize = max_t(int, bsize, 1024);
                bsize = min_t(int, bsize, NFSSVC_MAXBLKSIZE);
                bsize &= ~(1024-1);
                mutex_lock(&nfsd_mutex);
                if (nn->nfsd_serv) {
                        mutex_unlock(&nfsd_mutex);
                        return -EBUSY;
                }
                nfsd_max_blksize = bsize;
                mutex_unlock(&nfsd_mutex);
        }

        return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%d\n",
                                                        nfsd_max_blksize);
}

/*
 * write_maxconn - Set or report the current max number of connections
 *
 * Input:
 *                        buf:                ignored
 *                        size:                zero
 * OR
 *
 * Input:
 *                        buf:                C string containing an unsigned
 *                                        integer value representing the new
 *                                        number of max connections
 *                        size:                non-zero length of C string in @buf
 * Output:
 *        On success:        passed-in buffer filled with '\n'-terminated C string
 *                        containing numeric value of max_connections setting
 *                        for this net namespace;
 *                        return code is the size in bytes of the string
 *        On error:        return code is zero or a negative errno value
 */
static ssize_t write_maxconn(struct file *file, char *buf, size_t size)
{
        char *mesg = buf;
        struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id);
        unsigned int maxconn = nn->max_connections;

        if (size > 0) {
                int rv = get_uint(&mesg, &maxconn);

                if (rv)
                        return rv;
                trace_nfsd_ctl_maxconn(netns(file), maxconn);
                nn->max_connections = maxconn;
        }

        return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%u\n", maxconn);
}

#ifdef CONFIG_NFSD_V4
static ssize_t __nfsd4_write_time(struct file *file, char *buf, size_t size,
                                  time64_t *time, struct nfsd_net *nn)
{
        struct dentry *dentry = file_dentry(file);
        char *mesg = buf;
        int rv, i;

        if (size > 0) {
                if (nn->nfsd_serv)
                        return -EBUSY;
                rv = get_int(&mesg, &i);
                if (rv)
                        return rv;
                trace_nfsd_ctl_time(netns(file), dentry->d_name.name,
                                    dentry->d_name.len, i);

                /*
                 * Some sanity checking.  We don't have a reason for
                 * these particular numbers, but problems with the
                 * extremes are:
                 *        - Too short: the briefest network outage may
                 *          cause clients to lose all their locks.  Also,
                 *          the frequent polling may be wasteful.
                 *        - Too long: do you really want reboot recovery
                 *          to take more than an hour?  Or to make other
                 *          clients wait an hour before being able to
                 *          revoke a dead client's locks?
                 */
                if (i < 10 || i > 3600)
                        return -EINVAL;
                *time = i;
        }

        return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%lld\n", *time);
}

static ssize_t nfsd4_write_time(struct file *file, char *buf, size_t size,
                                time64_t *time, struct nfsd_net *nn)
{
        ssize_t rv;

        mutex_lock(&nfsd_mutex);
        rv = __nfsd4_write_time(file, buf, size, time, nn);
        mutex_unlock(&nfsd_mutex);
        return rv;
}

/*
 * write_leasetime - Set or report the current NFSv4 lease time
 *
 * Input:
 *                        buf:                ignored
 *                        size:                zero
 *
 * OR
 *
 * Input:
 *                        buf:                C string containing an unsigned
 *                                        integer value representing the new
 *                                        NFSv4 lease expiry time
 *                        size:                non-zero length of C string in @buf
 * Output:
 *        On success:        passed-in buffer filled with '\n'-terminated C
 *                        string containing unsigned integer value of the
 *                        current lease expiry time;
 *                        return code is the size in bytes of the string
 *        On error:        return code is zero or a negative errno value
 */
static ssize_t write_leasetime(struct file *file, char *buf, size_t size)
{
        struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id);
        return nfsd4_write_time(file, buf, size, &nn->nfsd4_lease, nn);
}

/*
 * write_gracetime - Set or report current NFSv4 grace period time
 *
 * As above, but sets the time of the NFSv4 grace period.
 *
 * Note this should never be set to less than the *previous*
 * lease-period time, but we don't try to enforce this.  (In the common
 * case (a new boot), we don't know what the previous lease time was
 * anyway.)
 */
static ssize_t write_gracetime(struct file *file, char *buf, size_t size)
{
        struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id);
        return nfsd4_write_time(file, buf, size, &nn->nfsd4_grace, nn);
}

#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
static ssize_t __write_recoverydir(struct file *file, char *buf, size_t size,
                                   struct nfsd_net *nn)
{
        char *mesg = buf;
        char *recdir;
        int len, status;

        if (size > 0) {
                if (nn->nfsd_serv)
                        return -EBUSY;
                if (size > PATH_MAX || buf[size-1] != '\n')
                        return -EINVAL;
                buf[size-1] = 0;

                recdir = mesg;
                len = qword_get(&mesg, recdir, size);
                if (len <= 0)
                        return -EINVAL;
                trace_nfsd_ctl_recoverydir(netns(file), recdir);

                status = nfs4_reset_recoverydir(recdir);
                if (status)
                        return status;
        }

        return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%s\n",
                                                        nfs4_recoverydir());
}

/*
 * write_recoverydir - Set or report the pathname of the recovery directory
 *
 * Input:
 *                        buf:                ignored
 *                        size:                zero
 *
 * OR
 *
 * Input:
 *                        buf:                C string containing the pathname
 *                                        of the directory on a local file
 *                                        system containing permanent NFSv4
 *                                        recovery data
 *                        size:                non-zero length of C string in @buf
 * Output:
 *        On success:        passed-in buffer filled with '\n'-terminated C string
 *                        containing the current recovery pathname setting;
 *                        return code is the size in bytes of the string
 *        On error:        return code is zero or a negative errno value
 */
static ssize_t write_recoverydir(struct file *file, char *buf, size_t size)
{
        ssize_t rv;
        struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id);

        mutex_lock(&nfsd_mutex);
        rv = __write_recoverydir(file, buf, size, nn);
        mutex_unlock(&nfsd_mutex);
        return rv;
}
#endif

/*
 * write_v4_end_grace - release grace period for nfsd's v4.x lock manager
 *
 * Input:
 *                        buf:                ignored
 *                        size:                zero
 * OR
 *
 * Input:
 *                        buf:                any value
 *                        size:                non-zero length of C string in @buf
 * Output:
 *                        passed-in buffer filled with "Y" or "N" with a newline
 *                        and NULL-terminated C string. This indicates whether
 *                        the grace period has ended in the current net
 *                        namespace. Return code is the size in bytes of the
 *                        string. Writing a string that starts with 'Y', 'y', or
 *                        '1' to the file will end the grace period for nfsd's v4
 *                        lock manager.
 */
static ssize_t write_v4_end_grace(struct file *file, char *buf, size_t size)
{
        struct nfsd_net *nn = net_generic(netns(file), nfsd_net_id);

        if (size > 0) {
                switch(buf[0]) {
                case 'Y':
                case 'y':
                case '1':
                        if (!nn->nfsd_serv)
                                return -EBUSY;
                        trace_nfsd_end_grace(netns(file));
                        nfsd4_end_grace(nn);
                        break;
                default:
                        return -EINVAL;
                }
        }

        return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%c\n",
                         nn->grace_ended ? 'Y' : 'N');
}

#endif

/*----------------------------------------------------------------------------*/
/*
 *        populating the filesystem.
 */

/* Basically copying rpc_get_inode. */
static struct inode *nfsd_get_inode(struct super_block *sb, umode_t mode)
{
        struct inode *inode = new_inode(sb);
        if (!inode)
                return NULL;
        /* Following advice from simple_fill_super documentation: */
        inode->i_ino = iunique(sb, NFSD_MaxReserved);
        inode->i_mode = mode;
        simple_inode_init_ts(inode);
        switch (mode & S_IFMT) {
        case S_IFDIR:
                inode->i_fop = &simple_dir_operations;
                inode->i_op = &simple_dir_inode_operations;
                inc_nlink(inode);
                break;
        case S_IFLNK:
                inode->i_op = &simple_symlink_inode_operations;
                break;
        default:
                break;
        }
        return inode;
}

static int __nfsd_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode, struct nfsdfs_client *ncl)
{
        struct inode *inode;

        inode = nfsd_get_inode(dir->i_sb, mode);
        if (!inode)
                return -ENOMEM;
        if (ncl) {
                inode->i_private = ncl;
                kref_get(&ncl->cl_ref);
        }
        d_add(dentry, inode);
        inc_nlink(dir);
        fsnotify_mkdir(dir, dentry);
        return 0;
}

static struct dentry *nfsd_mkdir(struct dentry *parent, struct nfsdfs_client *ncl, char *name)
{
        struct inode *dir = parent->d_inode;
        struct dentry *dentry;
        int ret = -ENOMEM;

        inode_lock(dir);
        dentry = d_alloc_name(parent, name);
        if (!dentry)
                goto out_err;
        ret = __nfsd_mkdir(d_inode(parent), dentry, S_IFDIR | 0600, ncl);
        if (ret)
                goto out_err;
out:
        inode_unlock(dir);
        return dentry;
out_err:
        dput(dentry);
        dentry = ERR_PTR(ret);
        goto out;
}

#if IS_ENABLED(CONFIG_SUNRPC_GSS)
static int __nfsd_symlink(struct inode *dir, struct dentry *dentry,
                          umode_t mode, const char *content)
{
        struct inode *inode;

        inode = nfsd_get_inode(dir->i_sb, mode);
        if (!inode)
                return -ENOMEM;

        inode->i_link = (char *)content;
        inode->i_size = strlen(content);

        d_add(dentry, inode);
        inc_nlink(dir);
        fsnotify_create(dir, dentry);
        return 0;
}

/*
 * @content is assumed to be a NUL-terminated string that lives
 * longer than the symlink itself.
 */
static void _nfsd_symlink(struct dentry *parent, const char *name,
                          const char *content)
{
        struct inode *dir = parent->d_inode;
        struct dentry *dentry;
        int ret;

        inode_lock(dir);
        dentry = d_alloc_name(parent, name);
        if (!dentry)
                goto out;
        ret = __nfsd_symlink(d_inode(parent), dentry, S_IFLNK | 0777, content);
        if (ret)
                dput(dentry);
out:
        inode_unlock(dir);
}
#else
static inline void _nfsd_symlink(struct dentry *parent, const char *name,
                                 const char *content)
{
}

#endif

static void clear_ncl(struct dentry *dentry)
{
        struct inode *inode = d_inode(dentry);
        struct nfsdfs_client *ncl = inode->i_private;

        spin_lock(&inode->i_lock);
        inode->i_private = NULL;
        spin_unlock(&inode->i_lock);
        kref_put(&ncl->cl_ref, ncl->cl_release);
}

struct nfsdfs_client *get_nfsdfs_client(struct inode *inode)
{
        struct nfsdfs_client *nc;

        spin_lock(&inode->i_lock);
        nc = inode->i_private;
        if (nc)
                kref_get(&nc->cl_ref);
        spin_unlock(&inode->i_lock);
        return nc;
}

/* XXX: cut'n'paste from simple_fill_super; figure out if we could share
 * code instead. */
static  int nfsdfs_create_files(struct dentry *root,
                                const struct tree_descr *files,
                                struct nfsdfs_client *ncl,
                                struct dentry **fdentries)
{
        struct inode *dir = d_inode(root);
        struct inode *inode;
        struct dentry *dentry;
        int i;

        inode_lock(dir);
        for (i = 0; files->name && files->name[0]; i++, files++) {
                dentry = d_alloc_name(root, files->name);
                if (!dentry)
                        goto out;
                inode = nfsd_get_inode(d_inode(root)->i_sb,
                                        S_IFREG | files->mode);
                if (!inode) {
                        dput(dentry);
                        goto out;
                }
                kref_get(&ncl->cl_ref);
                inode->i_fop = files->ops;
                inode->i_private = ncl;
                d_add(dentry, inode);
                fsnotify_create(dir, dentry);
                if (fdentries)
                        fdentries[i] = dentry;
        }
        inode_unlock(dir);
        return 0;
out:
        inode_unlock(dir);
        return -ENOMEM;
}

/* on success, returns positive number unique to that client. */
struct dentry *nfsd_client_mkdir(struct nfsd_net *nn,
                                 struct nfsdfs_client *ncl, u32 id,
                                 const struct tree_descr *files,
                                 struct dentry **fdentries)
{
        struct dentry *dentry;
        char name[11];
        int ret;

        sprintf(name, "%u", id);

        dentry = nfsd_mkdir(nn->nfsd_client_dir, ncl, name);
        if (IS_ERR(dentry)) /* XXX: tossing errors? */
                return NULL;
        ret = nfsdfs_create_files(dentry, files, ncl, fdentries);
        if (ret) {
                nfsd_client_rmdir(dentry);
                return NULL;
        }
        return dentry;
}

/* Taken from __rpc_rmdir: */
void nfsd_client_rmdir(struct dentry *dentry)
{
        simple_recursive_removal(dentry, clear_ncl);
}

static int nfsd_fill_super(struct super_block *sb, struct fs_context *fc)
{
        struct nfsd_net *nn = net_generic(current->nsproxy->net_ns,
                                                        nfsd_net_id);
        struct dentry *dentry;
        int ret;

        static const struct tree_descr nfsd_files[] = {
                [NFSD_List] = {"exports", &exports_nfsd_operations, S_IRUGO},
                /* Per-export io stats use same ops as exports file */
                [NFSD_Export_Stats] = {"export_stats", &exports_nfsd_operations, S_IRUGO},
                [NFSD_Export_features] = {"export_features",
                                        &export_features_fops, S_IRUGO},
                [NFSD_FO_UnlockIP] = {"unlock_ip",
                                        &transaction_ops, S_IWUSR|S_IRUSR},
                [NFSD_FO_UnlockFS] = {"unlock_filesystem",
                                        &transaction_ops, S_IWUSR|S_IRUSR},
                [NFSD_Fh] = {"filehandle", &transaction_ops, S_IWUSR|S_IRUSR},
                [NFSD_Threads] = {"threads", &transaction_ops, S_IWUSR|S_IRUSR},
                [NFSD_Pool_Threads] = {"pool_threads", &transaction_ops, S_IWUSR|S_IRUSR},
                [NFSD_Pool_Stats] = {"pool_stats", &pool_stats_operations, S_IRUGO},
                [NFSD_Reply_Cache_Stats] = {"reply_cache_stats",
                                        &nfsd_reply_cache_stats_fops, S_IRUGO},
                [NFSD_Versions] = {"versions", &transaction_ops, S_IWUSR|S_IRUSR},
                [NFSD_Ports] = {"portlist", &transaction_ops, S_IWUSR|S_IRUGO},
                [NFSD_MaxBlkSize] = {"max_block_size", &transaction_ops, S_IWUSR|S_IRUGO},
                [NFSD_MaxConnections] = {"max_connections", &transaction_ops, S_IWUSR|S_IRUGO},
                [NFSD_Filecache] = {"filecache", &nfsd_file_cache_stats_fops, S_IRUGO},
#ifdef CONFIG_NFSD_V4
                [NFSD_Leasetime] = {"nfsv4leasetime", &transaction_ops, S_IWUSR|S_IRUSR},
                [NFSD_Gracetime] = {"nfsv4gracetime", &transaction_ops, S_IWUSR|S_IRUSR},
#ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING
                [NFSD_RecoveryDir] = {"nfsv4recoverydir", &transaction_ops, S_IWUSR|S_IRUSR},
#endif
                [NFSD_V4EndGrace] = {"v4_end_grace", &transaction_ops, S_IWUSR|S_IRUGO},
#endif
                /* last one */ {""}
        };

        ret = simple_fill_super(sb, 0x6e667364, nfsd_files);
        if (ret)
                return ret;
        _nfsd_symlink(sb->s_root, "supported_krb5_enctypes",
                      "/proc/net/rpc/gss_krb5_enctypes");
        dentry = nfsd_mkdir(sb->s_root, NULL, "clients");
        if (IS_ERR(dentry))
                return PTR_ERR(dentry);
        nn->nfsd_client_dir = dentry;
        return 0;
}

static int nfsd_fs_get_tree(struct fs_context *fc)
{
        return get_tree_keyed(fc, nfsd_fill_super, get_net(fc->net_ns));
}

static void nfsd_fs_free_fc(struct fs_context *fc)
{
        if (fc->s_fs_info)
                put_net(fc->s_fs_info);
}

static const struct fs_context_operations nfsd_fs_context_ops = {
        .free                = nfsd_fs_free_fc,
        .get_tree        = nfsd_fs_get_tree,
};

static int nfsd_init_fs_context(struct fs_context *fc)
{
        put_user_ns(fc->user_ns);
        fc->user_ns = get_user_ns(fc->net_ns->user_ns);
        fc->ops = &nfsd_fs_context_ops;
        return 0;
}

static void nfsd_umount(struct super_block *sb)
{
        struct net *net = sb->s_fs_info;

        nfsd_shutdown_threads(net);

        kill_litter_super(sb);
        put_net(net);
}

static struct file_system_type nfsd_fs_type = {
        .owner                = THIS_MODULE,
        .name                = "nfsd",
        .init_fs_context = nfsd_init_fs_context,
        .kill_sb        = nfsd_umount,
};
MODULE_ALIAS_FS("nfsd");

#ifdef CONFIG_PROC_FS

static int exports_proc_open(struct inode *inode, struct file *file)
{
        return exports_net_open(current->nsproxy->net_ns, file);
}

static const struct proc_ops exports_proc_ops = {
        .proc_open        = exports_proc_open,
        .proc_read        = seq_read,
        .proc_lseek        = seq_lseek,
        .proc_release        = seq_release,
};

static int create_proc_exports_entry(void)
{
        struct proc_dir_entry *entry;

        entry = proc_mkdir("fs/nfs", NULL);
        if (!entry)
                return -ENOMEM;
        entry = proc_create("exports", 0, entry, &exports_proc_ops);
        if (!entry) {
                remove_proc_entry("fs/nfs", NULL);
                return -ENOMEM;
        }
        return 0;
}
#else /* CONFIG_PROC_FS */
static int create_proc_exports_entry(void)
{
        return 0;
}
#endif

unsigned int nfsd_net_id;

static int nfsd_genl_rpc_status_compose_msg(struct sk_buff *skb,
                                            struct netlink_callback *cb,
                                            struct nfsd_genl_rqstp *rqstp)
{
        void *hdr;
        u32 i;

        hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
                          &nfsd_nl_family, 0, NFSD_CMD_RPC_STATUS_GET);
        if (!hdr)
                return -ENOBUFS;

        if (nla_put_be32(skb, NFSD_A_RPC_STATUS_XID, rqstp->rq_xid) ||
            nla_put_u32(skb, NFSD_A_RPC_STATUS_FLAGS, rqstp->rq_flags) ||
            nla_put_u32(skb, NFSD_A_RPC_STATUS_PROG, rqstp->rq_prog) ||
            nla_put_u32(skb, NFSD_A_RPC_STATUS_PROC, rqstp->rq_proc) ||
            nla_put_u8(skb, NFSD_A_RPC_STATUS_VERSION, rqstp->rq_vers) ||
            nla_put_s64(skb, NFSD_A_RPC_STATUS_SERVICE_TIME,
                        ktime_to_us(rqstp->rq_stime),
                        NFSD_A_RPC_STATUS_PAD))
                return -ENOBUFS;

        switch (rqstp->rq_saddr.sa_family) {
        case AF_INET: {
                const struct sockaddr_in *s_in, *d_in;

                s_in = (const struct sockaddr_in *)&rqstp->rq_saddr;
                d_in = (const struct sockaddr_in *)&rqstp->rq_daddr;
                if (nla_put_in_addr(skb, NFSD_A_RPC_STATUS_SADDR4,
                                    s_in->sin_addr.s_addr) ||
                    nla_put_in_addr(skb, NFSD_A_RPC_STATUS_DADDR4,
                                    d_in->sin_addr.s_addr) ||
                    nla_put_be16(skb, NFSD_A_RPC_STATUS_SPORT,
                                 s_in->sin_port) ||
                    nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT,
                                 d_in->sin_port))
                        return -ENOBUFS;
                break;
        }
        case AF_INET6: {
                const struct sockaddr_in6 *s_in, *d_in;

                s_in = (const struct sockaddr_in6 *)&rqstp->rq_saddr;
                d_in = (const struct sockaddr_in6 *)&rqstp->rq_daddr;
                if (nla_put_in6_addr(skb, NFSD_A_RPC_STATUS_SADDR6,
                                     &s_in->sin6_addr) ||
                    nla_put_in6_addr(skb, NFSD_A_RPC_STATUS_DADDR6,
                                     &d_in->sin6_addr) ||
                    nla_put_be16(skb, NFSD_A_RPC_STATUS_SPORT,
                                 s_in->sin6_port) ||
                    nla_put_be16(skb, NFSD_A_RPC_STATUS_DPORT,
                                 d_in->sin6_port))
                        return -ENOBUFS;
                break;
        }
        }

        for (i = 0; i < rqstp->rq_opcnt; i++)
                if (nla_put_u32(skb, NFSD_A_RPC_STATUS_COMPOUND_OPS,
                                rqstp->rq_opnum[i]))
                        return -ENOBUFS;

        genlmsg_end(skb, hdr);
        return 0;
}

/**
 * nfsd_nl_rpc_status_get_dumpit - Handle rpc_status_get dumpit
 * @skb: reply buffer
 * @cb: netlink metadata and command arguments
 *
 * Returns the size of the reply or a negative errno.
 */
int nfsd_nl_rpc_status_get_dumpit(struct sk_buff *skb,
                                  struct netlink_callback *cb)
{
        int i, ret, rqstp_index = 0;
        struct nfsd_net *nn;

        mutex_lock(&nfsd_mutex);

        nn = net_generic(sock_net(skb->sk), nfsd_net_id);
        if (!nn->nfsd_serv) {
                ret = -ENODEV;
                goto out_unlock;
        }

        rcu_read_lock();

        for (i = 0; i < nn->nfsd_serv->sv_nrpools; i++) {
                struct svc_rqst *rqstp;

                if (i < cb->args[0]) /* already consumed */
                        continue;

                rqstp_index = 0;
                list_for_each_entry_rcu(rqstp,
                                &nn->nfsd_serv->sv_pools[i].sp_all_threads,
                                rq_all) {
                        struct nfsd_genl_rqstp genl_rqstp;
                        unsigned int status_counter;

                        if (rqstp_index++ < cb->args[1]) /* already consumed */
                                continue;
                        /*
                         * Acquire rq_status_counter before parsing the rqst
                         * fields. rq_status_counter is set to an odd value in
                         * order to notify the consumers the rqstp fields are
                         * meaningful.
                         */
                        status_counter =
                                smp_load_acquire(&rqstp->rq_status_counter);
                        if (!(status_counter & 1))
                                continue;

                        genl_rqstp.rq_xid = rqstp->rq_xid;
                        genl_rqstp.rq_flags = rqstp->rq_flags;
                        genl_rqstp.rq_vers = rqstp->rq_vers;
                        genl_rqstp.rq_prog = rqstp->rq_prog;
                        genl_rqstp.rq_proc = rqstp->rq_proc;
                        genl_rqstp.rq_stime = rqstp->rq_stime;
                        genl_rqstp.rq_opcnt = 0;
                        memcpy(&genl_rqstp.rq_daddr, svc_daddr(rqstp),
                               sizeof(struct sockaddr));
                        memcpy(&genl_rqstp.rq_saddr, svc_addr(rqstp),
                               sizeof(struct sockaddr));

#ifdef CONFIG_NFSD_V4
                        if (rqstp->rq_vers == NFS4_VERSION &&
                            rqstp->rq_proc == NFSPROC4_COMPOUND) {
                                /* NFSv4 compound */
                                struct nfsd4_compoundargs *args;
                                int j;

                                args = rqstp->rq_argp;
                                genl_rqstp.rq_opcnt = args->opcnt;
                                for (j = 0; j < genl_rqstp.rq_opcnt; j++)
                                        genl_rqstp.rq_opnum[j] =
                                                args->ops[j].opnum;
                        }
#endif /* CONFIG_NFSD_V4 */

                        /*
                         * Acquire rq_status_counter before reporting the rqst
                         * fields to the user.
                         */
                        if (smp_load_acquire(&rqstp->rq_status_counter) !=
                            status_counter)
                                continue;

                        ret = nfsd_genl_rpc_status_compose_msg(skb, cb,
                                                               &genl_rqstp);
                        if (ret)
                                goto out;
                }
        }

        cb->args[0] = i;
        cb->args[1] = rqstp_index;
        ret = skb->len;
out:
        rcu_read_unlock();
out_unlock:
        mutex_unlock(&nfsd_mutex);

        return ret;
}

/**
 * nfsd_nl_threads_set_doit - set the number of running threads
 * @skb: reply buffer
 * @info: netlink metadata and command arguments
 *
 * Return 0 on success or a negative errno.
 */
int nfsd_nl_threads_set_doit(struct sk_buff *skb, struct genl_info *info)
{
        int *nthreads, count = 0, nrpools, i, ret = -EOPNOTSUPP, rem;
        struct net *net = genl_info_net(info);
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        const struct nlattr *attr;
        const char *scope = NULL;

        if (GENL_REQ_ATTR_CHECK(info, NFSD_A_SERVER_THREADS))
                return -EINVAL;

        /* count number of SERVER_THREADS values */
        nlmsg_for_each_attr(attr, info->nlhdr, GENL_HDRLEN, rem) {
                if (nla_type(attr) == NFSD_A_SERVER_THREADS)
                        count++;
        }

        mutex_lock(&nfsd_mutex);

        nrpools = max(count, nfsd_nrpools(net));
        nthreads = kcalloc(nrpools, sizeof(int), GFP_KERNEL);
        if (!nthreads) {
                ret = -ENOMEM;
                goto out_unlock;
        }

        i = 0;
        nlmsg_for_each_attr(attr, info->nlhdr, GENL_HDRLEN, rem) {
                if (nla_type(attr) == NFSD_A_SERVER_THREADS) {
                        nthreads[i++] = nla_get_u32(attr);
                        if (i >= nrpools)
                                break;
                }
        }

        if (info->attrs[NFSD_A_SERVER_GRACETIME] ||
            info->attrs[NFSD_A_SERVER_LEASETIME] ||
            info->attrs[NFSD_A_SERVER_SCOPE]) {
                ret = -EBUSY;
                if (nn->nfsd_serv && nn->nfsd_serv->sv_nrthreads)
                        goto out_unlock;

                ret = -EINVAL;
                attr = info->attrs[NFSD_A_SERVER_GRACETIME];
                if (attr) {
                        u32 gracetime = nla_get_u32(attr);

                        if (gracetime < 10 || gracetime > 3600)
                                goto out_unlock;

                        nn->nfsd4_grace = gracetime;
                }

                attr = info->attrs[NFSD_A_SERVER_LEASETIME];
                if (attr) {
                        u32 leasetime = nla_get_u32(attr);

                        if (leasetime < 10 || leasetime > 3600)
                                goto out_unlock;

                        nn->nfsd4_lease = leasetime;
                }

                attr = info->attrs[NFSD_A_SERVER_SCOPE];
                if (attr)
                        scope = nla_data(attr);
        }

        ret = nfsd_svc(nrpools, nthreads, net, get_current_cred(), scope);
        if (ret > 0)
                ret = 0;
out_unlock:
        mutex_unlock(&nfsd_mutex);
        kfree(nthreads);
        return ret;
}

/**
 * nfsd_nl_threads_get_doit - get the number of running threads
 * @skb: reply buffer
 * @info: netlink metadata and command arguments
 *
 * Return 0 on success or a negative errno.
 */
int nfsd_nl_threads_get_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        void *hdr;
        int err;

        skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb)
                return -ENOMEM;

        hdr = genlmsg_iput(skb, info);
        if (!hdr) {
                err = -EMSGSIZE;
                goto err_free_msg;
        }

        mutex_lock(&nfsd_mutex);

        err = nla_put_u32(skb, NFSD_A_SERVER_GRACETIME,
                          nn->nfsd4_grace) ||
              nla_put_u32(skb, NFSD_A_SERVER_LEASETIME,
                          nn->nfsd4_lease) ||
              nla_put_string(skb, NFSD_A_SERVER_SCOPE,
                          nn->nfsd_name);
        if (err)
                goto err_unlock;

        if (nn->nfsd_serv) {
                int i;

                for (i = 0; i < nfsd_nrpools(net); ++i) {
                        struct svc_pool *sp = &nn->nfsd_serv->sv_pools[i];

                        err = nla_put_u32(skb, NFSD_A_SERVER_THREADS,
                                          sp->sp_nrthreads);
                        if (err)
                                goto err_unlock;
                }
        } else {
                err = nla_put_u32(skb, NFSD_A_SERVER_THREADS, 0);
                if (err)
                        goto err_unlock;
        }

        mutex_unlock(&nfsd_mutex);

        genlmsg_end(skb, hdr);

        return genlmsg_reply(skb, info);

err_unlock:
        mutex_unlock(&nfsd_mutex);
err_free_msg:
        nlmsg_free(skb);

        return err;
}

/**
 * nfsd_nl_version_set_doit - set the nfs enabled versions
 * @skb: reply buffer
 * @info: netlink metadata and command arguments
 *
 * Return 0 on success or a negative errno.
 */
int nfsd_nl_version_set_doit(struct sk_buff *skb, struct genl_info *info)
{
        const struct nlattr *attr;
        struct nfsd_net *nn;
        int i, rem;

        if (GENL_REQ_ATTR_CHECK(info, NFSD_A_SERVER_PROTO_VERSION))
                return -EINVAL;

        mutex_lock(&nfsd_mutex);

        nn = net_generic(genl_info_net(info), nfsd_net_id);
        if (nn->nfsd_serv) {
                mutex_unlock(&nfsd_mutex);
                return -EBUSY;
        }

        /* clear current supported versions. */
        nfsd_vers(nn, 2, NFSD_CLEAR);
        nfsd_vers(nn, 3, NFSD_CLEAR);
        for (i = 0; i <= NFSD_SUPPORTED_MINOR_VERSION; i++)
                nfsd_minorversion(nn, i, NFSD_CLEAR);

        nlmsg_for_each_attr(attr, info->nlhdr, GENL_HDRLEN, rem) {
                struct nlattr *tb[NFSD_A_VERSION_MAX + 1];
                u32 major, minor = 0;
                bool enabled;

                if (nla_type(attr) != NFSD_A_SERVER_PROTO_VERSION)
                        continue;

                if (nla_parse_nested(tb, NFSD_A_VERSION_MAX, attr,
                                     nfsd_version_nl_policy, info->extack) < 0)
                        continue;

                if (!tb[NFSD_A_VERSION_MAJOR])
                        continue;

                major = nla_get_u32(tb[NFSD_A_VERSION_MAJOR]);
                if (tb[NFSD_A_VERSION_MINOR])
                        minor = nla_get_u32(tb[NFSD_A_VERSION_MINOR]);

                enabled = nla_get_flag(tb[NFSD_A_VERSION_ENABLED]);

                switch (major) {
                case 4:
                        nfsd_minorversion(nn, minor, enabled ? NFSD_SET : NFSD_CLEAR);
                        break;
                case 3:
                case 2:
                        if (!minor)
                                nfsd_vers(nn, major, enabled ? NFSD_SET : NFSD_CLEAR);
                        break;
                default:
                        break;
                }
        }

        mutex_unlock(&nfsd_mutex);

        return 0;
}

/**
 * nfsd_nl_version_get_doit - get the enabled status for all supported nfs versions
 * @skb: reply buffer
 * @info: netlink metadata and command arguments
 *
 * Return 0 on success or a negative errno.
 */
int nfsd_nl_version_get_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct nfsd_net *nn;
        int i, err;
        void *hdr;

        skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb)
                return -ENOMEM;

        hdr = genlmsg_iput(skb, info);
        if (!hdr) {
                err = -EMSGSIZE;
                goto err_free_msg;
        }

        mutex_lock(&nfsd_mutex);
        nn = net_generic(genl_info_net(info), nfsd_net_id);

        for (i = 2; i <= 4; i++) {
                int j;

                for (j = 0; j <= NFSD_SUPPORTED_MINOR_VERSION; j++) {
                        struct nlattr *attr;

                        /* Don't record any versions the kernel doesn't have
                         * compiled in
                         */
                        if (!nfsd_support_version(i))
                                continue;

                        /* NFSv{2,3} does not support minor numbers */
                        if (i < 4 && j)
                                continue;

                        attr = nla_nest_start(skb,
                                              NFSD_A_SERVER_PROTO_VERSION);
                        if (!attr) {
                                err = -EINVAL;
                                goto err_nfsd_unlock;
                        }

                        if (nla_put_u32(skb, NFSD_A_VERSION_MAJOR, i) ||
                            nla_put_u32(skb, NFSD_A_VERSION_MINOR, j)) {
                                err = -EINVAL;
                                goto err_nfsd_unlock;
                        }

                        /* Set the enabled flag if the version is enabled */
                        if (nfsd_vers(nn, i, NFSD_TEST) &&
                            (i < 4 || nfsd_minorversion(nn, j, NFSD_TEST)) &&
                            nla_put_flag(skb, NFSD_A_VERSION_ENABLED)) {
                                err = -EINVAL;
                                goto err_nfsd_unlock;
                        }

                        nla_nest_end(skb, attr);
                }
        }

        mutex_unlock(&nfsd_mutex);
        genlmsg_end(skb, hdr);

        return genlmsg_reply(skb, info);

err_nfsd_unlock:
        mutex_unlock(&nfsd_mutex);
err_free_msg:
        nlmsg_free(skb);

        return err;
}

/**
 * nfsd_nl_listener_set_doit - set the nfs running sockets
 * @skb: reply buffer
 * @info: netlink metadata and command arguments
 *
 * Return 0 on success or a negative errno.
 */
int nfsd_nl_listener_set_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        struct svc_xprt *xprt, *tmp;
        const struct nlattr *attr;
        struct svc_serv *serv;
        LIST_HEAD(permsocks);
        struct nfsd_net *nn;
        int err, rem;

        mutex_lock(&nfsd_mutex);

        err = nfsd_create_serv(net);
        if (err) {
                mutex_unlock(&nfsd_mutex);
                return err;
        }

        nn = net_generic(net, nfsd_net_id);
        serv = nn->nfsd_serv;

        spin_lock_bh(&serv->sv_lock);

        /* Move all of the old listener sockets to a temp list */
        list_splice_init(&serv->sv_permsocks, &permsocks);

        /*
         * Walk the list of server_socks from userland and move any that match
         * back to sv_permsocks
         */
        nlmsg_for_each_attr(attr, info->nlhdr, GENL_HDRLEN, rem) {
                struct nlattr *tb[NFSD_A_SOCK_MAX + 1];
                const char *xcl_name;
                struct sockaddr *sa;

                if (nla_type(attr) != NFSD_A_SERVER_SOCK_ADDR)
                        continue;

                if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr,
                                     nfsd_sock_nl_policy, info->extack) < 0)
                        continue;

                if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME])
                        continue;

                if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa))
                        continue;

                xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]);
                sa = nla_data(tb[NFSD_A_SOCK_ADDR]);

                /* Put back any matching sockets */
                list_for_each_entry_safe(xprt, tmp, &permsocks, xpt_list) {
                        /* This shouldn't be possible */
                        if (WARN_ON_ONCE(xprt->xpt_net != net)) {
                                list_move(&xprt->xpt_list, &serv->sv_permsocks);
                                continue;
                        }

                        /* If everything matches, put it back */
                        if (!strcmp(xprt->xpt_class->xcl_name, xcl_name) &&
                            rpc_cmp_addr_port(sa, (struct sockaddr *)&xprt->xpt_local)) {
                                list_move(&xprt->xpt_list, &serv->sv_permsocks);
                                break;
                        }
                }
        }

        /* For now, no removing old sockets while server is running */
        if (serv->sv_nrthreads && !list_empty(&permsocks)) {
                list_splice_init(&permsocks, &serv->sv_permsocks);
                spin_unlock_bh(&serv->sv_lock);
                err = -EBUSY;
                goto out_unlock_mtx;
        }

        /* Close the remaining sockets on the permsocks list */
        while (!list_empty(&permsocks)) {
                xprt = list_first_entry(&permsocks, struct svc_xprt, xpt_list);
                list_move(&xprt->xpt_list, &serv->sv_permsocks);

                /*
                 * Newly-created sockets are born with the BUSY bit set. Clear
                 * it if there are no threads, since nothing can pick it up
                 * in that case.
                 */
                if (!serv->sv_nrthreads)
                        clear_bit(XPT_BUSY, &xprt->xpt_flags);

                set_bit(XPT_CLOSE, &xprt->xpt_flags);
                spin_unlock_bh(&serv->sv_lock);
                svc_xprt_close(xprt);
                spin_lock_bh(&serv->sv_lock);
        }

        spin_unlock_bh(&serv->sv_lock);

        /* walk list of addrs again, open any that still don't exist */
        nlmsg_for_each_attr(attr, info->nlhdr, GENL_HDRLEN, rem) {
                struct nlattr *tb[NFSD_A_SOCK_MAX + 1];
                const char *xcl_name;
                struct sockaddr *sa;
                int ret;

                if (nla_type(attr) != NFSD_A_SERVER_SOCK_ADDR)
                        continue;

                if (nla_parse_nested(tb, NFSD_A_SOCK_MAX, attr,
                                     nfsd_sock_nl_policy, info->extack) < 0)
                        continue;

                if (!tb[NFSD_A_SOCK_ADDR] || !tb[NFSD_A_SOCK_TRANSPORT_NAME])
                        continue;

                if (nla_len(tb[NFSD_A_SOCK_ADDR]) < sizeof(*sa))
                        continue;

                xcl_name = nla_data(tb[NFSD_A_SOCK_TRANSPORT_NAME]);
                sa = nla_data(tb[NFSD_A_SOCK_ADDR]);

                xprt = svc_find_listener(serv, xcl_name, net, sa);
                if (xprt) {
                        svc_xprt_put(xprt);
                        continue;
                }

                ret = svc_xprt_create_from_sa(serv, xcl_name, net, sa, 0,
                                              get_current_cred());
                /* always save the latest error */
                if (ret < 0)
                        err = ret;
        }

        if (!serv->sv_nrthreads && list_empty(&nn->nfsd_serv->sv_permsocks))
                nfsd_destroy_serv(net);

out_unlock_mtx:
        mutex_unlock(&nfsd_mutex);

        return err;
}

/**
 * nfsd_nl_listener_get_doit - get the nfs running listeners
 * @skb: reply buffer
 * @info: netlink metadata and command arguments
 *
 * Return 0 on success or a negative errno.
 */
int nfsd_nl_listener_get_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct svc_xprt *xprt;
        struct svc_serv *serv;
        struct nfsd_net *nn;
        void *hdr;
        int err;

        skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb)
                return -ENOMEM;

        hdr = genlmsg_iput(skb, info);
        if (!hdr) {
                err = -EMSGSIZE;
                goto err_free_msg;
        }

        mutex_lock(&nfsd_mutex);
        nn = net_generic(genl_info_net(info), nfsd_net_id);

        /* no nfs server? Just send empty socket list */
        if (!nn->nfsd_serv)
                goto out_unlock_mtx;

        serv = nn->nfsd_serv;
        spin_lock_bh(&serv->sv_lock);
        list_for_each_entry(xprt, &serv->sv_permsocks, xpt_list) {
                struct nlattr *attr;

                attr = nla_nest_start(skb, NFSD_A_SERVER_SOCK_ADDR);
                if (!attr) {
                        err = -EINVAL;
                        goto err_serv_unlock;
                }

                if (nla_put_string(skb, NFSD_A_SOCK_TRANSPORT_NAME,
                                   xprt->xpt_class->xcl_name) ||
                    nla_put(skb, NFSD_A_SOCK_ADDR,
                            sizeof(struct sockaddr_storage),
                            &xprt->xpt_local)) {
                        err = -EINVAL;
                        goto err_serv_unlock;
                }

                nla_nest_end(skb, attr);
        }
        spin_unlock_bh(&serv->sv_lock);
out_unlock_mtx:
        mutex_unlock(&nfsd_mutex);
        genlmsg_end(skb, hdr);

        return genlmsg_reply(skb, info);

err_serv_unlock:
        spin_unlock_bh(&serv->sv_lock);
        mutex_unlock(&nfsd_mutex);
err_free_msg:
        nlmsg_free(skb);

        return err;
}

/**
 * nfsd_nl_pool_mode_set_doit - set the number of running threads
 * @skb: reply buffer
 * @info: netlink metadata and command arguments
 *
 * Return 0 on success or a negative errno.
 */
int nfsd_nl_pool_mode_set_doit(struct sk_buff *skb, struct genl_info *info)
{
        const struct nlattr *attr;

        if (GENL_REQ_ATTR_CHECK(info, NFSD_A_POOL_MODE_MODE))
                return -EINVAL;

        attr = info->attrs[NFSD_A_POOL_MODE_MODE];
        return sunrpc_set_pool_mode(nla_data(attr));
}

/**
 * nfsd_nl_pool_mode_get_doit - get info about pool_mode
 * @skb: reply buffer
 * @info: netlink metadata and command arguments
 *
 * Return 0 on success or a negative errno.
 */
int nfsd_nl_pool_mode_get_doit(struct sk_buff *skb, struct genl_info *info)
{
        struct net *net = genl_info_net(info);
        char buf[16];
        void *hdr;
        int err;

        if (sunrpc_get_pool_mode(buf, ARRAY_SIZE(buf)) >= ARRAY_SIZE(buf))
                return -ERANGE;

        skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
        if (!skb)
                return -ENOMEM;

        err = -EMSGSIZE;
        hdr = genlmsg_iput(skb, info);
        if (!hdr)
                goto err_free_msg;

        err = nla_put_string(skb, NFSD_A_POOL_MODE_MODE, buf) |
              nla_put_u32(skb, NFSD_A_POOL_MODE_NPOOLS, nfsd_nrpools(net));
        if (err)
                goto err_free_msg;

        genlmsg_end(skb, hdr);
        return genlmsg_reply(skb, info);

err_free_msg:
        nlmsg_free(skb);
        return err;
}

/**
 * nfsd_net_init - Prepare the nfsd_net portion of a new net namespace
 * @net: a freshly-created network namespace
 *
 * This information stays around as long as the network namespace is
 * alive whether or not there is an NFSD instance running in the
 * namespace.
 *
 * Returns zero on success, or a negative errno otherwise.
 */
static __net_init int nfsd_net_init(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);
        int retval;
        int i;

        retval = nfsd_export_init(net);
        if (retval)
                goto out_export_error;
        retval = nfsd_idmap_init(net);
        if (retval)
                goto out_idmap_error;
        retval = percpu_counter_init_many(nn->counter, 0, GFP_KERNEL,
                                          NFSD_STATS_COUNTERS_NUM);
        if (retval)
                goto out_repcache_error;
        memset(&nn->nfsd_svcstats, 0, sizeof(nn->nfsd_svcstats));
        nn->nfsd_svcstats.program = &nfsd_programs[0];
        for (i = 0; i < sizeof(nn->nfsd_versions); i++)
                nn->nfsd_versions[i] = nfsd_support_version(i);
        for (i = 0; i < sizeof(nn->nfsd4_minorversions); i++)
                nn->nfsd4_minorversions[i] = nfsd_support_version(4);
        nn->nfsd_info.mutex = &nfsd_mutex;
        nn->nfsd_serv = NULL;
        nfsd4_init_leases_net(nn);
        get_random_bytes(&nn->siphash_key, sizeof(nn->siphash_key));
        seqlock_init(&nn->writeverf_lock);
        nfsd_proc_stat_init(net);
#if IS_ENABLED(CONFIG_NFS_LOCALIO)
        INIT_LIST_HEAD(&nn->local_clients);
#endif
        return 0;

out_repcache_error:
        nfsd_idmap_shutdown(net);
out_idmap_error:
        nfsd_export_shutdown(net);
out_export_error:
        return retval;
}

#if IS_ENABLED(CONFIG_NFS_LOCALIO)
/**
 * nfsd_net_pre_exit - Disconnect localio clients from net namespace
 * @net: a network namespace that is about to be destroyed
 *
 * This invalidated ->net pointers held by localio clients
 * while they can still safely access nn->counter.
 */
static __net_exit void nfsd_net_pre_exit(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        nfs_uuid_invalidate_clients(&nn->local_clients);
}
#endif

/**
 * nfsd_net_exit - Release the nfsd_net portion of a net namespace
 * @net: a network namespace that is about to be destroyed
 *
 */
static __net_exit void nfsd_net_exit(struct net *net)
{
        struct nfsd_net *nn = net_generic(net, nfsd_net_id);

        nfsd_proc_stat_shutdown(net);
        percpu_counter_destroy_many(nn->counter, NFSD_STATS_COUNTERS_NUM);
        nfsd_idmap_shutdown(net);
        nfsd_export_shutdown(net);
}

static struct pernet_operations nfsd_net_ops = {
        .init = nfsd_net_init,
#if IS_ENABLED(CONFIG_NFS_LOCALIO)
        .pre_exit = nfsd_net_pre_exit,
#endif
        .exit = nfsd_net_exit,
        .id   = &nfsd_net_id,
        .size = sizeof(struct nfsd_net),
};

static int __init init_nfsd(void)
{
        int retval;

        retval = nfsd4_init_slabs();
        if (retval)
                return retval;
        retval = nfsd4_init_pnfs();
        if (retval)
                goto out_free_slabs;
        retval = nfsd_drc_slab_create();
        if (retval)
                goto out_free_pnfs;
        nfsd_lockd_init();        /* lockd->nfsd callbacks */
        retval = create_proc_exports_entry();
        if (retval)
                goto out_free_lockd;
        retval = register_pernet_subsys(&nfsd_net_ops);
        if (retval < 0)
                goto out_free_exports;
        retval = register_cld_notifier();
        if (retval)
                goto out_free_subsys;
        retval = nfsd4_create_laundry_wq();
        if (retval)
                goto out_free_cld;
        retval = register_filesystem(&nfsd_fs_type);
        if (retval)
                goto out_free_all;
        retval = genl_register_family(&nfsd_nl_family);
        if (retval)
                goto out_free_all;
        nfsd_localio_ops_init();

        return 0;
out_free_all:
        nfsd4_destroy_laundry_wq();
out_free_cld:
        unregister_cld_notifier();
out_free_subsys:
        unregister_pernet_subsys(&nfsd_net_ops);
out_free_exports:
        remove_proc_entry("fs/nfs/exports", NULL);
        remove_proc_entry("fs/nfs", NULL);
out_free_lockd:
        nfsd_lockd_shutdown();
        nfsd_drc_slab_free();
out_free_pnfs:
        nfsd4_exit_pnfs();
out_free_slabs:
        nfsd4_free_slabs();
        return retval;
}

static void __exit exit_nfsd(void)
{
        genl_unregister_family(&nfsd_nl_family);
        unregister_filesystem(&nfsd_fs_type);
        nfsd4_destroy_laundry_wq();
        unregister_cld_notifier();
        unregister_pernet_subsys(&nfsd_net_ops);
        nfsd_drc_slab_free();
        remove_proc_entry("fs/nfs/exports", NULL);
        remove_proc_entry("fs/nfs", NULL);
        nfsd_lockd_shutdown();
        nfsd4_free_slabs();
        nfsd4_exit_pnfs();
}

MODULE_AUTHOR("Olaf Kirch <okir@monad.swb.de>");
MODULE_DESCRIPTION("In-kernel NFS server");
MODULE_LICENSE("GPL");
module_init(init_nfsd)
module_exit(exit_nfsd)









































































   12 



   12 


   12 

   12 







   11 


   10 
   11 










   12 

















   12 








   12 


































   12 



   12 
















   11 








   11 





   11 















































































































    2 




    1 












    1 



    6 




    1 



    2 



    3 
    1 
    1 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
/*
 * Copyright (c) 2016 Intel Corporation
 *
 * Permission to use, copy, modify, distribute, and sell this software and its
 * documentation for any purpose is hereby granted without fee, provided that
 * the above copyright notice appear in all copies and that both that copyright
 * notice and this permission notice appear in supporting documentation, and
 * that the name of the copyright holders not be used in advertising or
 * publicity pertaining to distribution of the software without specific,
 * written prior permission.  The copyright holders make no representations
 * about the suitability of this software for any purpose.  It is provided "as
 * is" without express or implied warranty.
 *
 * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
 * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
 * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
 * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
 * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
 * OF THIS SOFTWARE.
 */

#include <linux/export.h>

#include <drm/drm_bridge.h>
#include <drm/drm_device.h>
#include <drm/drm_drv.h>
#include <drm/drm_encoder.h>
#include <drm/drm_managed.h>
#include <drm/drm_print.h>

#include "drm_crtc_internal.h"
#include "drm_internal.h"

/**
 * DOC: overview
 *
 * Encoders represent the connecting element between the CRTC (as the overall
 * pixel pipeline, represented by &struct drm_crtc) and the connectors (as the
 * generic sink entity, represented by &struct drm_connector). An encoder takes
 * pixel data from a CRTC and converts it to a format suitable for any attached
 * connector. Encoders are objects exposed to userspace, originally to allow
 * userspace to infer cloning and connector/CRTC restrictions. Unfortunately
 * almost all drivers get this wrong, making the uabi pretty much useless. On
 * top of that the exposed restrictions are too simple for today's hardware, and
 * the recommended way to infer restrictions is by using the
 * DRM_MODE_ATOMIC_TEST_ONLY flag for the atomic IOCTL.
 *
 * Otherwise encoders aren't used in the uapi at all (any modeset request from
 * userspace directly connects a connector with a CRTC), drivers are therefore
 * free to use them however they wish. Modeset helper libraries make strong use
 * of encoders to facilitate code sharing. But for more complex settings it is
 * usually better to move shared code into a separate &drm_bridge. Compared to
 * encoders, bridges also have the benefit of being purely an internal
 * abstraction since they are not exposed to userspace at all.
 *
 * Encoders are initialized with drm_encoder_init() and cleaned up using
 * drm_encoder_cleanup().
 */
static const struct drm_prop_enum_list drm_encoder_enum_list[] = {
        { DRM_MODE_ENCODER_NONE, "None" },
        { DRM_MODE_ENCODER_DAC, "DAC" },
        { DRM_MODE_ENCODER_TMDS, "TMDS" },
        { DRM_MODE_ENCODER_LVDS, "LVDS" },
        { DRM_MODE_ENCODER_TVDAC, "TV" },
        { DRM_MODE_ENCODER_VIRTUAL, "Virtual" },
        { DRM_MODE_ENCODER_DSI, "DSI" },
        { DRM_MODE_ENCODER_DPMST, "DP MST" },
        { DRM_MODE_ENCODER_DPI, "DPI" },
};

int drm_encoder_register_all(struct drm_device *dev)
{
        struct drm_encoder *encoder;
        int ret = 0;

        drm_for_each_encoder(encoder, dev) {
                drm_debugfs_encoder_add(encoder);

                if (encoder->funcs && encoder->funcs->late_register)
                        ret = encoder->funcs->late_register(encoder);
                if (ret)
                        return ret;
        }

        return 0;
}

void drm_encoder_unregister_all(struct drm_device *dev)
{
        struct drm_encoder *encoder;

        drm_for_each_encoder(encoder, dev) {
                if (encoder->funcs && encoder->funcs->early_unregister)
                        encoder->funcs->early_unregister(encoder);
                drm_debugfs_encoder_remove(encoder);
        }
}

__printf(5, 0)
static int __drm_encoder_init(struct drm_device *dev,
                              struct drm_encoder *encoder,
                              const struct drm_encoder_funcs *funcs,
                              int encoder_type, const char *name, va_list ap)
{
        int ret;

        /* encoder index is used with 32bit bitmasks */
        if (WARN_ON(dev->mode_config.num_encoder >= 32))
                return -EINVAL;

        ret = drm_mode_object_add(dev, &encoder->base, DRM_MODE_OBJECT_ENCODER);
        if (ret)
                return ret;

        encoder->dev = dev;
        encoder->encoder_type = encoder_type;
        encoder->funcs = funcs;
        if (name) {
                encoder->name = kvasprintf(GFP_KERNEL, name, ap);
        } else {
                encoder->name = kasprintf(GFP_KERNEL, "%s-%d",
                                          drm_encoder_enum_list[encoder_type].name,
                                          encoder->base.id);
        }
        if (!encoder->name) {
                ret = -ENOMEM;
                goto out_put;
        }

        INIT_LIST_HEAD(&encoder->bridge_chain);
        list_add_tail(&encoder->head, &dev->mode_config.encoder_list);
        encoder->index = dev->mode_config.num_encoder++;

out_put:
        if (ret)
                drm_mode_object_unregister(dev, &encoder->base);

        return ret;
}

/**
 * drm_encoder_init - Init a preallocated encoder
 * @dev: drm device
 * @encoder: the encoder to init
 * @funcs: callbacks for this encoder
 * @encoder_type: user visible type of the encoder
 * @name: printf style format string for the encoder name, or NULL for default name
 *
 * Initializes a preallocated encoder. Encoder should be subclassed as part of
 * driver encoder objects. At driver unload time the driver's
 * &drm_encoder_funcs.destroy hook should call drm_encoder_cleanup() and kfree()
 * the encoder structure. The encoder structure should not be allocated with
 * devm_kzalloc().
 *
 * Note: consider using drmm_encoder_alloc() or drmm_encoder_init()
 * instead of drm_encoder_init() to let the DRM managed resource
 * infrastructure take care of cleanup and deallocation.
 *
 * Returns:
 * Zero on success, error code on failure.
 */
int drm_encoder_init(struct drm_device *dev,
                     struct drm_encoder *encoder,
                     const struct drm_encoder_funcs *funcs,
                     int encoder_type, const char *name, ...)
{
        va_list ap;
        int ret;

        WARN_ON(!funcs->destroy);

        va_start(ap, name);
        ret = __drm_encoder_init(dev, encoder, funcs, encoder_type, name, ap);
        va_end(ap);

        return ret;
}
EXPORT_SYMBOL(drm_encoder_init);

/**
 * drm_encoder_cleanup - cleans up an initialised encoder
 * @encoder: encoder to cleanup
 *
 * Cleans up the encoder but doesn't free the object.
 */
void drm_encoder_cleanup(struct drm_encoder *encoder)
{
        struct drm_device *dev = encoder->dev;
        struct drm_bridge *bridge, *next;

        /* Note that the encoder_list is considered to be static; should we
         * remove the drm_encoder at runtime we would have to decrement all
         * the indices on the drm_encoder after us in the encoder_list.
         */

        list_for_each_entry_safe(bridge, next, &encoder->bridge_chain,
                                 chain_node)
                drm_bridge_detach(bridge);

        drm_mode_object_unregister(dev, &encoder->base);
        kfree(encoder->name);
        list_del(&encoder->head);
        dev->mode_config.num_encoder--;

        memset(encoder, 0, sizeof(*encoder));
}
EXPORT_SYMBOL(drm_encoder_cleanup);

static void drmm_encoder_alloc_release(struct drm_device *dev, void *ptr)
{
        struct drm_encoder *encoder = ptr;

        if (WARN_ON(!encoder->dev))
                return;

        drm_encoder_cleanup(encoder);
}

__printf(5, 0)
static int __drmm_encoder_init(struct drm_device *dev,
                               struct drm_encoder *encoder,
                               const struct drm_encoder_funcs *funcs,
                               int encoder_type,
                               const char *name,
                               va_list args)
{
        int ret;

        if (drm_WARN_ON(dev, funcs && funcs->destroy))
                return -EINVAL;

        ret = __drm_encoder_init(dev, encoder, funcs, encoder_type, name, args);
        if (ret)
                return ret;

        ret = drmm_add_action_or_reset(dev, drmm_encoder_alloc_release, encoder);
        if (ret)
                return ret;

        return 0;
}

void *__drmm_encoder_alloc(struct drm_device *dev, size_t size, size_t offset,
                           const struct drm_encoder_funcs *funcs,
                           int encoder_type, const char *name, ...)
{
        void *container;
        struct drm_encoder *encoder;
        va_list ap;
        int ret;

        container = drmm_kzalloc(dev, size, GFP_KERNEL);
        if (!container)
                return ERR_PTR(-ENOMEM);

        encoder = container + offset;

        va_start(ap, name);
        ret = __drmm_encoder_init(dev, encoder, funcs, encoder_type, name, ap);
        va_end(ap);
        if (ret)
                return ERR_PTR(ret);

        return container;
}
EXPORT_SYMBOL(__drmm_encoder_alloc);

/**
 * drmm_encoder_init - Initialize a preallocated encoder
 * @dev: drm device
 * @encoder: the encoder to init
 * @funcs: callbacks for this encoder (optional)
 * @encoder_type: user visible type of the encoder
 * @name: printf style format string for the encoder name, or NULL for default name
 *
 * Initializes a preallocated encoder. Encoder should be subclassed as
 * part of driver encoder objects. Cleanup is automatically handled
 * through registering drm_encoder_cleanup() with drmm_add_action(). The
 * encoder structure should be allocated with drmm_kzalloc().
 *
 * The @drm_encoder_funcs.destroy hook must be NULL.
 *
 * Returns:
 * Zero on success, error code on failure.
 */
int drmm_encoder_init(struct drm_device *dev, struct drm_encoder *encoder,
                      const struct drm_encoder_funcs *funcs,
                      int encoder_type, const char *name, ...)
{
        va_list ap;
        int ret;

        va_start(ap, name);
        ret = __drmm_encoder_init(dev, encoder, funcs, encoder_type, name, ap);
        va_end(ap);
        if (ret)
                return ret;

        return 0;
}
EXPORT_SYMBOL(drmm_encoder_init);

static struct drm_crtc *drm_encoder_get_crtc(struct drm_encoder *encoder)
{
        struct drm_connector *connector;
        struct drm_device *dev = encoder->dev;
        bool uses_atomic = false;
        struct drm_connector_list_iter conn_iter;

        /* For atomic drivers only state objects are synchronously updated and
         * protected by modeset locks, so check those first. */
        drm_connector_list_iter_begin(dev, &conn_iter);
        drm_for_each_connector_iter(connector, &conn_iter) {
                if (!connector->state)
                        continue;

                uses_atomic = true;

                if (connector->state->best_encoder != encoder)
                        continue;

                drm_connector_list_iter_end(&conn_iter);
                return connector->state->crtc;
        }
        drm_connector_list_iter_end(&conn_iter);

        /* Don't return stale data (e.g. pending async disable). */
        if (uses_atomic)
                return NULL;

        return encoder->crtc;
}

int drm_mode_getencoder(struct drm_device *dev, void *data,
                        struct drm_file *file_priv)
{
        struct drm_mode_get_encoder *enc_resp = data;
        struct drm_encoder *encoder;
        struct drm_crtc *crtc;

        if (!drm_core_check_feature(dev, DRIVER_MODESET))
                return -EOPNOTSUPP;

        encoder = drm_encoder_find(dev, file_priv, enc_resp->encoder_id);
        if (!encoder)
                return -ENOENT;

        drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
        crtc = drm_encoder_get_crtc(encoder);
        if (crtc && drm_lease_held(file_priv, crtc->base.id))
                enc_resp->crtc_id = crtc->base.id;
        else
                enc_resp->crtc_id = 0;
        drm_modeset_unlock(&dev->mode_config.connection_mutex);

        enc_resp->encoder_type = encoder->encoder_type;
        enc_resp->encoder_id = encoder->base.id;
        enc_resp->possible_crtcs = drm_lease_filter_crtcs(file_priv,
                                                          encoder->possible_crtcs);
        enc_resp->possible_clones = encoder->possible_clones;

        return 0;
}




















































































  651 




  652 


























  255 



  253 














































































































































































































































































































































































  253 





























































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright(c) 2017 Intel Corporation. All rights reserved.
 */
#include <linux/pagemap.h>
#include <linux/module.h>
#include <linux/mount.h>
#include <linux/pseudo_fs.h>
#include <linux/magic.h>
#include <linux/pfn_t.h>
#include <linux/cdev.h>
#include <linux/slab.h>
#include <linux/uio.h>
#include <linux/dax.h>
#include <linux/fs.h>
#include <linux/cacheinfo.h>
#include "dax-private.h"

/**
 * struct dax_device - anchor object for dax services
 * @inode: core vfs
 * @cdev: optional character interface for "device dax"
 * @private: dax driver private data
 * @flags: state and boolean properties
 * @ops: operations for this device
 * @holder_data: holder of a dax_device: could be filesystem or mapped device
 * @holder_ops: operations for the inner holder
 */
struct dax_device {
        struct inode inode;
        struct cdev cdev;
        void *private;
        unsigned long flags;
        const struct dax_operations *ops;
        void *holder_data;
        const struct dax_holder_operations *holder_ops;
};

static dev_t dax_devt;
DEFINE_STATIC_SRCU(dax_srcu);
static struct vfsmount *dax_mnt;
static DEFINE_IDA(dax_minor_ida);
static struct kmem_cache *dax_cache __read_mostly;
static struct super_block *dax_superblock __read_mostly;

int dax_read_lock(void)
{
        return srcu_read_lock(&dax_srcu);
}
EXPORT_SYMBOL_GPL(dax_read_lock);

void dax_read_unlock(int id)
{
        srcu_read_unlock(&dax_srcu, id);
}
EXPORT_SYMBOL_GPL(dax_read_unlock);

#if defined(CONFIG_BLOCK) && defined(CONFIG_FS_DAX)
#include <linux/blkdev.h>

static DEFINE_XARRAY(dax_hosts);

int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk)
{
        return xa_insert(&dax_hosts, (unsigned long)disk, dax_dev, GFP_KERNEL);
}
EXPORT_SYMBOL_GPL(dax_add_host);

void dax_remove_host(struct gendisk *disk)
{
        xa_erase(&dax_hosts, (unsigned long)disk);
}
EXPORT_SYMBOL_GPL(dax_remove_host);

/**
 * fs_dax_get_by_bdev() - temporary lookup mechanism for filesystem-dax
 * @bdev: block device to find a dax_device for
 * @start_off: returns the byte offset into the dax_device that @bdev starts
 * @holder: filesystem or mapped device inside the dax_device
 * @ops: operations for the inner holder
 */
struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off,
                void *holder, const struct dax_holder_operations *ops)
{
        struct dax_device *dax_dev;
        u64 part_size;
        int id;

        if (!blk_queue_dax(bdev->bd_disk->queue))
                return NULL;

        *start_off = get_start_sect(bdev) * SECTOR_SIZE;
        part_size = bdev_nr_sectors(bdev) * SECTOR_SIZE;
        if (*start_off % PAGE_SIZE || part_size % PAGE_SIZE) {
                pr_info("%pg: error: unaligned partition for dax\n", bdev);
                return NULL;
        }

        id = dax_read_lock();
        dax_dev = xa_load(&dax_hosts, (unsigned long)bdev->bd_disk);
        if (!dax_dev || !dax_alive(dax_dev) || !igrab(&dax_dev->inode))
                dax_dev = NULL;
        else if (holder) {
                if (!cmpxchg(&dax_dev->holder_data, NULL, holder))
                        dax_dev->holder_ops = ops;
                else
                        dax_dev = NULL;
        }
        dax_read_unlock(id);

        return dax_dev;
}
EXPORT_SYMBOL_GPL(fs_dax_get_by_bdev);

void fs_put_dax(struct dax_device *dax_dev, void *holder)
{
        if (dax_dev && holder &&
            cmpxchg(&dax_dev->holder_data, holder, NULL) == holder)
                dax_dev->holder_ops = NULL;
        put_dax(dax_dev);
}
EXPORT_SYMBOL_GPL(fs_put_dax);
#endif /* CONFIG_BLOCK && CONFIG_FS_DAX */

enum dax_device_flags {
        /* !alive + rcu grace period == no new operations / mappings */
        DAXDEV_ALIVE,
        /* gate whether dax_flush() calls the low level flush routine */
        DAXDEV_WRITE_CACHE,
        /* flag to check if device supports synchronous flush */
        DAXDEV_SYNC,
        /* do not leave the caches dirty after writes */
        DAXDEV_NOCACHE,
        /* handle CPU fetch exceptions during reads */
        DAXDEV_NOMC,
};

/**
 * dax_direct_access() - translate a device pgoff to an absolute pfn
 * @dax_dev: a dax_device instance representing the logical memory range
 * @pgoff: offset in pages from the start of the device to translate
 * @nr_pages: number of consecutive pages caller can handle relative to @pfn
 * @mode: indicator on normal access or recovery write
 * @kaddr: output parameter that returns a virtual address mapping of pfn
 * @pfn: output parameter that returns an absolute pfn translation of @pgoff
 *
 * Return: negative errno if an error occurs, otherwise the number of
 * pages accessible at the device relative @pgoff.
 */
long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages,
                enum dax_access_mode mode, void **kaddr, pfn_t *pfn)
{
        long avail;

        if (!dax_dev)
                return -EOPNOTSUPP;

        if (!dax_alive(dax_dev))
                return -ENXIO;

        if (nr_pages < 0)
                return -EINVAL;

        avail = dax_dev->ops->direct_access(dax_dev, pgoff, nr_pages,
                        mode, kaddr, pfn);
        if (!avail)
                return -ERANGE;
        return min(avail, nr_pages);
}
EXPORT_SYMBOL_GPL(dax_direct_access);

size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
                size_t bytes, struct iov_iter *i)
{
        if (!dax_alive(dax_dev))
                return 0;

        /*
         * The userspace address for the memory copy has already been validated
         * via access_ok() in vfs_write, so use the 'no check' version to bypass
         * the HARDENED_USERCOPY overhead.
         */
        if (test_bit(DAXDEV_NOCACHE, &dax_dev->flags))
                return _copy_from_iter_flushcache(addr, bytes, i);
        return _copy_from_iter(addr, bytes, i);
}

size_t dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr,
                size_t bytes, struct iov_iter *i)
{
        if (!dax_alive(dax_dev))
                return 0;

        /*
         * The userspace address for the memory copy has already been validated
         * via access_ok() in vfs_red, so use the 'no check' version to bypass
         * the HARDENED_USERCOPY overhead.
         */
        if (test_bit(DAXDEV_NOMC, &dax_dev->flags))
                return _copy_mc_to_iter(addr, bytes, i);
        return _copy_to_iter(addr, bytes, i);
}

int dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
                        size_t nr_pages)
{
        int ret;

        if (!dax_alive(dax_dev))
                return -ENXIO;
        /*
         * There are no callers that want to zero more than one page as of now.
         * Once users are there, this check can be removed after the
         * device mapper code has been updated to split ranges across targets.
         */
        if (nr_pages != 1)
                return -EIO;

        ret = dax_dev->ops->zero_page_range(dax_dev, pgoff, nr_pages);
        return dax_mem2blk_err(ret);
}
EXPORT_SYMBOL_GPL(dax_zero_page_range);

size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff,
                void *addr, size_t bytes, struct iov_iter *iter)
{
        if (!dax_dev->ops->recovery_write)
                return 0;
        return dax_dev->ops->recovery_write(dax_dev, pgoff, addr, bytes, iter);
}
EXPORT_SYMBOL_GPL(dax_recovery_write);

int dax_holder_notify_failure(struct dax_device *dax_dev, u64 off,
                              u64 len, int mf_flags)
{
        int rc, id;

        id = dax_read_lock();
        if (!dax_alive(dax_dev)) {
                rc = -ENXIO;
                goto out;
        }

        if (!dax_dev->holder_ops) {
                rc = -EOPNOTSUPP;
                goto out;
        }

        rc = dax_dev->holder_ops->notify_failure(dax_dev, off, len, mf_flags);
out:
        dax_read_unlock(id);
        return rc;
}
EXPORT_SYMBOL_GPL(dax_holder_notify_failure);

#ifdef CONFIG_ARCH_HAS_PMEM_API
void arch_wb_cache_pmem(void *addr, size_t size);
void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
{
        if (unlikely(!dax_write_cache_enabled(dax_dev)))
                return;

        arch_wb_cache_pmem(addr, size);
}
#else
void dax_flush(struct dax_device *dax_dev, void *addr, size_t size)
{
}
#endif
EXPORT_SYMBOL_GPL(dax_flush);

void dax_write_cache(struct dax_device *dax_dev, bool wc)
{
        if (wc)
                set_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
        else
                clear_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
}
EXPORT_SYMBOL_GPL(dax_write_cache);

bool dax_write_cache_enabled(struct dax_device *dax_dev)
{
        return test_bit(DAXDEV_WRITE_CACHE, &dax_dev->flags);
}
EXPORT_SYMBOL_GPL(dax_write_cache_enabled);

bool dax_synchronous(struct dax_device *dax_dev)
{
        return test_bit(DAXDEV_SYNC, &dax_dev->flags);
}
EXPORT_SYMBOL_GPL(dax_synchronous);

void set_dax_synchronous(struct dax_device *dax_dev)
{
        set_bit(DAXDEV_SYNC, &dax_dev->flags);
}
EXPORT_SYMBOL_GPL(set_dax_synchronous);

void set_dax_nocache(struct dax_device *dax_dev)
{
        set_bit(DAXDEV_NOCACHE, &dax_dev->flags);
}
EXPORT_SYMBOL_GPL(set_dax_nocache);

void set_dax_nomc(struct dax_device *dax_dev)
{
        set_bit(DAXDEV_NOMC, &dax_dev->flags);
}
EXPORT_SYMBOL_GPL(set_dax_nomc);

bool dax_alive(struct dax_device *dax_dev)
{
        lockdep_assert_held(&dax_srcu);
        return test_bit(DAXDEV_ALIVE, &dax_dev->flags);
}
EXPORT_SYMBOL_GPL(dax_alive);

/*
 * Note, rcu is not protecting the liveness of dax_dev, rcu is ensuring
 * that any fault handlers or operations that might have seen
 * dax_alive(), have completed.  Any operations that start after
 * synchronize_srcu() has run will abort upon seeing !dax_alive().
 *
 * Note, because alloc_dax() returns an ERR_PTR() on error, callers
 * typically store its result into a local variable in order to check
 * the result. Therefore, care must be taken to populate the struct
 * device dax_dev field make sure the dax_dev is not leaked.
 */
void kill_dax(struct dax_device *dax_dev)
{
        if (!dax_dev)
                return;

        if (dax_dev->holder_data != NULL)
                dax_holder_notify_failure(dax_dev, 0, U64_MAX,
                                MF_MEM_PRE_REMOVE);

        clear_bit(DAXDEV_ALIVE, &dax_dev->flags);
        synchronize_srcu(&dax_srcu);

        /* clear holder data */
        dax_dev->holder_ops = NULL;
        dax_dev->holder_data = NULL;
}
EXPORT_SYMBOL_GPL(kill_dax);

void run_dax(struct dax_device *dax_dev)
{
        set_bit(DAXDEV_ALIVE, &dax_dev->flags);
}
EXPORT_SYMBOL_GPL(run_dax);

static struct inode *dax_alloc_inode(struct super_block *sb)
{
        struct dax_device *dax_dev;
        struct inode *inode;

        dax_dev = alloc_inode_sb(sb, dax_cache, GFP_KERNEL);
        if (!dax_dev)
                return NULL;

        inode = &dax_dev->inode;
        inode->i_rdev = 0;
        return inode;
}

static struct dax_device *to_dax_dev(struct inode *inode)
{
        return container_of(inode, struct dax_device, inode);
}

static void dax_free_inode(struct inode *inode)
{
        struct dax_device *dax_dev = to_dax_dev(inode);
        if (inode->i_rdev)
                ida_free(&dax_minor_ida, iminor(inode));
        kmem_cache_free(dax_cache, dax_dev);
}

static void dax_destroy_inode(struct inode *inode)
{
        struct dax_device *dax_dev = to_dax_dev(inode);
        WARN_ONCE(test_bit(DAXDEV_ALIVE, &dax_dev->flags),
                        "kill_dax() must be called before final iput()\n");
}

static const struct super_operations dax_sops = {
        .statfs = simple_statfs,
        .alloc_inode = dax_alloc_inode,
        .destroy_inode = dax_destroy_inode,
        .free_inode = dax_free_inode,
        .drop_inode = generic_delete_inode,
};

static int dax_init_fs_context(struct fs_context *fc)
{
        struct pseudo_fs_context *ctx = init_pseudo(fc, DAXFS_MAGIC);
        if (!ctx)
                return -ENOMEM;
        ctx->ops = &dax_sops;
        return 0;
}

static struct file_system_type dax_fs_type = {
        .name                = "dax",
        .init_fs_context = dax_init_fs_context,
        .kill_sb        = kill_anon_super,
};

static int dax_test(struct inode *inode, void *data)
{
        dev_t devt = *(dev_t *) data;

        return inode->i_rdev == devt;
}

static int dax_set(struct inode *inode, void *data)
{
        dev_t devt = *(dev_t *) data;

        inode->i_rdev = devt;
        return 0;
}

static struct dax_device *dax_dev_get(dev_t devt)
{
        struct dax_device *dax_dev;
        struct inode *inode;

        inode = iget5_locked(dax_superblock, hash_32(devt + DAXFS_MAGIC, 31),
                        dax_test, dax_set, &devt);

        if (!inode)
                return NULL;

        dax_dev = to_dax_dev(inode);
        if (inode->i_state & I_NEW) {
                set_bit(DAXDEV_ALIVE, &dax_dev->flags);
                inode->i_cdev = &dax_dev->cdev;
                inode->i_mode = S_IFCHR;
                inode->i_flags = S_DAX;
                mapping_set_gfp_mask(&inode->i_data, GFP_USER);
                unlock_new_inode(inode);
        }

        return dax_dev;
}

struct dax_device *alloc_dax(void *private, const struct dax_operations *ops)
{
        struct dax_device *dax_dev;
        dev_t devt;
        int minor;

        /*
         * Unavailable on architectures with virtually aliased data caches,
         * except for device-dax (NULL operations pointer), which does
         * not use aliased mappings from the kernel.
         */
        if (ops && cpu_dcache_is_aliasing())
                return ERR_PTR(-EOPNOTSUPP);

        if (WARN_ON_ONCE(ops && !ops->zero_page_range))
                return ERR_PTR(-EINVAL);

        minor = ida_alloc_max(&dax_minor_ida, MINORMASK, GFP_KERNEL);
        if (minor < 0)
                return ERR_PTR(-ENOMEM);

        devt = MKDEV(MAJOR(dax_devt), minor);
        dax_dev = dax_dev_get(devt);
        if (!dax_dev)
                goto err_dev;

        dax_dev->ops = ops;
        dax_dev->private = private;
        return dax_dev;

 err_dev:
        ida_free(&dax_minor_ida, minor);
        return ERR_PTR(-ENOMEM);
}
EXPORT_SYMBOL_GPL(alloc_dax);

void put_dax(struct dax_device *dax_dev)
{
        if (!dax_dev)
                return;
        iput(&dax_dev->inode);
}
EXPORT_SYMBOL_GPL(put_dax);

/**
 * dax_holder() - obtain the holder of a dax device
 * @dax_dev: a dax_device instance
 *
 * Return: the holder's data which represents the holder if registered,
 * otherwize NULL.
 */
void *dax_holder(struct dax_device *dax_dev)
{
        return dax_dev->holder_data;
}
EXPORT_SYMBOL_GPL(dax_holder);

/**
 * inode_dax: convert a public inode into its dax_dev
 * @inode: An inode with i_cdev pointing to a dax_dev
 *
 * Note this is not equivalent to to_dax_dev() which is for private
 * internal use where we know the inode filesystem type == dax_fs_type.
 */
struct dax_device *inode_dax(struct inode *inode)
{
        struct cdev *cdev = inode->i_cdev;

        return container_of(cdev, struct dax_device, cdev);
}
EXPORT_SYMBOL_GPL(inode_dax);

struct inode *dax_inode(struct dax_device *dax_dev)
{
        return &dax_dev->inode;
}
EXPORT_SYMBOL_GPL(dax_inode);

void *dax_get_private(struct dax_device *dax_dev)
{
        if (!test_bit(DAXDEV_ALIVE, &dax_dev->flags))
                return NULL;
        return dax_dev->private;
}
EXPORT_SYMBOL_GPL(dax_get_private);

static void init_once(void *_dax_dev)
{
        struct dax_device *dax_dev = _dax_dev;
        struct inode *inode = &dax_dev->inode;

        memset(dax_dev, 0, sizeof(*dax_dev));
        inode_init_once(inode);
}

static int dax_fs_init(void)
{
        int rc;

        dax_cache = kmem_cache_create("dax_cache", sizeof(struct dax_device), 0,
                        SLAB_HWCACHE_ALIGN | SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT,
                        init_once);
        if (!dax_cache)
                return -ENOMEM;

        dax_mnt = kern_mount(&dax_fs_type);
        if (IS_ERR(dax_mnt)) {
                rc = PTR_ERR(dax_mnt);
                goto err_mount;
        }
        dax_superblock = dax_mnt->mnt_sb;

        return 0;

 err_mount:
        kmem_cache_destroy(dax_cache);

        return rc;
}

static void dax_fs_exit(void)
{
        kern_unmount(dax_mnt);
        rcu_barrier();
        kmem_cache_destroy(dax_cache);
}

static int __init dax_core_init(void)
{
        int rc;

        rc = dax_fs_init();
        if (rc)
                return rc;

        rc = alloc_chrdev_region(&dax_devt, 0, MINORMASK+1, "dax");
        if (rc)
                goto err_chrdev;

        rc = dax_bus_init();
        if (rc)
                goto err_bus;
        return 0;

err_bus:
        unregister_chrdev_region(dax_devt, MINORMASK+1);
err_chrdev:
        dax_fs_exit();
        return 0;
}

static void __exit dax_core_exit(void)
{
        dax_bus_exit();
        unregister_chrdev_region(dax_devt, MINORMASK+1);
        ida_destroy(&dax_minor_ida);
        dax_fs_exit();
}

MODULE_AUTHOR("Intel Corporation");
MODULE_DESCRIPTION("DAX: direct access to differentiated memory");
MODULE_LICENSE("GPL v2");
subsys_initcall(dax_core_init);
module_exit(dax_core_exit);





















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    4 

    5 

















    5 
    4 

















    5 
    4 















    9 






    5 
    4 

























    5 

















    2 
















    5 

















    5 
    6 






   10 



























    9 





    9 
    9 

    9 

    9 


    4 
    5 







    9 


    9 

































    7 


    7 










    2 



    5 

    7 




















































































































































































   23 




















   23 









   23 


    1 

    9 
   14 


    3 
   20 
    2 
   21 



    1 






   14 
    7 

















    6 

    1 


    7 















    9 














   10 


    9 



    7 




    9 

    7 
    6 
































    6 
    6 




    6 






    6 

    6 




















   12 







    4 

    3 
















    2 
















    3 





    2 

















    4 

    4 

















   42 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
// SPDX-License-Identifier: GPL-2.0
/*
 * IPVS         An implementation of the IP virtual server support for the
 *              LINUX operating system.  IPVS is now implemented as a module
 *              over the NetFilter framework. IPVS can be used to build a
 *              high-performance and highly available server based on a
 *              cluster of servers.
 *
 * Version 1,   is capable of handling both version 0 and 1 messages.
 *              Version 0 is the plain old format.
 *              Note Version 0 receivers will just drop Ver 1 messages.
 *              Version 1 is capable of handle IPv6, Persistence data,
 *              time-outs, and firewall marks.
 *              In ver.1 "ip_vs_sync_conn_options" will be sent in netw. order.
 *              Ver. 0 can be turned on by sysctl -w net.ipv4.vs.sync_version=0
 *
 * Definitions  Message: is a complete datagram
 *              Sync_conn: is a part of a Message
 *              Param Data is an option to a Sync_conn.
 *
 * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
 *
 * ip_vs_sync:  sync connection info from master load balancer to backups
 *              through multicast
 *
 * Changes:
 *        Alexandre Cassen        :        Added master & backup support at a time.
 *        Alexandre Cassen        :        Added SyncID support for incoming sync
 *                                        messages filtering.
 *        Justin Ossevoort        :        Fix endian problem on sync message size.
 *        Hans Schillstrom        :        Added Version 1: i.e. IPv6,
 *                                        Persistence support, fwmark and time-out.
 */

#define KMSG_COMPONENT "IPVS"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/inetdevice.h>
#include <linux/net.h>
#include <linux/completion.h>
#include <linux/delay.h>
#include <linux/skbuff.h>
#include <linux/in.h>
#include <linux/igmp.h>                 /* for ip_mc_join_group */
#include <linux/udp.h>
#include <linux/err.h>
#include <linux/kthread.h>
#include <linux/wait.h>
#include <linux/kernel.h>
#include <linux/sched/signal.h>

#include <asm/unaligned.h>                /* Used for ntoh_seq and hton_seq */

#include <net/ip.h>
#include <net/sock.h>

#include <net/ip_vs.h>

#define IP_VS_SYNC_GROUP 0xe0000051    /* multicast addr - 224.0.0.81 */
#define IP_VS_SYNC_PORT  8848          /* multicast port */

#define SYNC_PROTO_VER  1                /* Protocol version in header */

static struct lock_class_key __ipvs_sync_key;
/*
 *        IPVS sync connection entry
 *        Version 0, i.e. original version.
 */
struct ip_vs_sync_conn_v0 {
        __u8                        reserved;

        /* Protocol, addresses and port numbers */
        __u8                        protocol;       /* Which protocol (TCP/UDP) */
        __be16                        cport;
        __be16                  vport;
        __be16                  dport;
        __be32                  caddr;          /* client address */
        __be32                  vaddr;          /* virtual address */
        __be32                  daddr;          /* destination address */

        /* Flags and state transition */
        __be16                  flags;          /* status flags */
        __be16                  state;          /* state info */

        /* The sequence options start here */
};

struct ip_vs_sync_conn_options {
        struct ip_vs_seq        in_seq;         /* incoming seq. struct */
        struct ip_vs_seq        out_seq;        /* outgoing seq. struct */
};

/*
     Sync Connection format (sync_conn)

       0                   1                   2                   3
       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |    Type       |    Protocol   | Ver.  |        Size           |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                             Flags                             |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |            State              |         cport                 |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |            vport              |         dport                 |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                             fwmark                            |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                             timeout  (in sec.)                |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                              ...                              |
      |                        IP-Addresses  (v4 or v6)               |
      |                              ...                              |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  Optional Parameters.
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      | Param. Type    | Param. Length |   Param. data                |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+                               |
      |                              ...                              |
      |                               +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                               | Param Type    | Param. Length |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                           Param  data                         |
      |         Last Param data should be padded for 32 bit alignment |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
*/

/*
 *  Type 0, IPv4 sync connection format
 */
struct ip_vs_sync_v4 {
        __u8                        type;
        __u8                        protocol;        /* Which protocol (TCP/UDP) */
        __be16                        ver_size;        /* Version msb 4 bits */
        /* Flags and state transition */
        __be32                        flags;                /* status flags */
        __be16                        state;                /* state info         */
        /* Protocol, addresses and port numbers */
        __be16                        cport;
        __be16                        vport;
        __be16                        dport;
        __be32                        fwmark;                /* Firewall mark from skb */
        __be32                        timeout;        /* cp timeout */
        __be32                        caddr;                /* client address */
        __be32                        vaddr;                /* virtual address */
        __be32                        daddr;                /* destination address */
        /* The sequence options start here */
        /* PE data padded to 32bit alignment after seq. options */
};
/*
 * Type 2 messages IPv6
 */
struct ip_vs_sync_v6 {
        __u8                        type;
        __u8                        protocol;        /* Which protocol (TCP/UDP) */
        __be16                        ver_size;        /* Version msb 4 bits */
        /* Flags and state transition */
        __be32                        flags;                /* status flags */
        __be16                        state;                /* state info         */
        /* Protocol, addresses and port numbers */
        __be16                        cport;
        __be16                        vport;
        __be16                        dport;
        __be32                        fwmark;                /* Firewall mark from skb */
        __be32                        timeout;        /* cp timeout */
        struct in6_addr                caddr;                /* client address */
        struct in6_addr                vaddr;                /* virtual address */
        struct in6_addr                daddr;                /* destination address */
        /* The sequence options start here */
        /* PE data padded to 32bit alignment after seq. options */
};

union ip_vs_sync_conn {
        struct ip_vs_sync_v4        v4;
        struct ip_vs_sync_v6        v6;
};

/* Bits in Type field in above */
#define STYPE_INET6                0
#define STYPE_F_INET6                (1 << STYPE_INET6)

#define SVER_SHIFT                12                /* Shift to get version */
#define SVER_MASK                0x0fff                /* Mask to strip version */

#define IPVS_OPT_SEQ_DATA        1
#define IPVS_OPT_PE_DATA        2
#define IPVS_OPT_PE_NAME        3
#define IPVS_OPT_PARAM                7

#define IPVS_OPT_F_SEQ_DATA        (1 << (IPVS_OPT_SEQ_DATA-1))
#define IPVS_OPT_F_PE_DATA        (1 << (IPVS_OPT_PE_DATA-1))
#define IPVS_OPT_F_PE_NAME        (1 << (IPVS_OPT_PE_NAME-1))
#define IPVS_OPT_F_PARAM        (1 << (IPVS_OPT_PARAM-1))

struct ip_vs_sync_thread_data {
        struct task_struct *task;
        struct netns_ipvs *ipvs;
        struct socket *sock;
        char *buf;
        int id;
};

/* Version 0 definition of packet sizes */
#define SIMPLE_CONN_SIZE  (sizeof(struct ip_vs_sync_conn_v0))
#define FULL_CONN_SIZE  \
(sizeof(struct ip_vs_sync_conn_v0) + sizeof(struct ip_vs_sync_conn_options))


/*
  The master mulitcasts messages (Datagrams) to the backup load balancers
  in the following format.

 Version 1:
  Note, first byte should be Zero, so ver 0 receivers will drop the packet.

       0                   1                   2                   3
       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |      0        |    SyncID     |            Size               |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |  Count Conns  |    Version    |    Reserved, set to Zero      |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                                                               |
      |                    IPVS Sync Connection (1)                   |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                            .                                  |
      ~                            .                                  ~
      |                            .                                  |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                                                               |
      |                    IPVS Sync Connection (n)                   |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

 Version 0 Header
       0                   1                   2                   3
       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |  Count Conns  |    SyncID     |            Size               |
      +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
      |                    IPVS Sync Connection (1)                   |
*/

/* Version 0 header */
struct ip_vs_sync_mesg_v0 {
        __u8                    nr_conns;
        __u8                    syncid;
        __be16                  size;

        /* ip_vs_sync_conn entries start here */
};

/* Version 1 header */
struct ip_vs_sync_mesg {
        __u8                        reserved;        /* must be zero */
        __u8                        syncid;
        __be16                        size;
        __u8                        nr_conns;
        __s8                        version;        /* SYNC_PROTO_VER  */
        __u16                        spare;
        /* ip_vs_sync_conn entries start here */
};

union ipvs_sockaddr {
        struct sockaddr_in        in;
        struct sockaddr_in6        in6;
};

struct ip_vs_sync_buff {
        struct list_head        list;
        unsigned long           firstuse;

        /* pointers for the message data */
        struct ip_vs_sync_mesg  *mesg;
        unsigned char           *head;
        unsigned char           *end;
};

/*
 * Copy of struct ip_vs_seq
 * From unaligned network order to aligned host order
 */
static void ntoh_seq(struct ip_vs_seq *no, struct ip_vs_seq *ho)
{
        memset(ho, 0, sizeof(*ho));
        ho->init_seq       = get_unaligned_be32(&no->init_seq);
        ho->delta          = get_unaligned_be32(&no->delta);
        ho->previous_delta = get_unaligned_be32(&no->previous_delta);
}

/*
 * Copy of struct ip_vs_seq
 * From Aligned host order to unaligned network order
 */
static void hton_seq(struct ip_vs_seq *ho, struct ip_vs_seq *no)
{
        put_unaligned_be32(ho->init_seq, &no->init_seq);
        put_unaligned_be32(ho->delta, &no->delta);
        put_unaligned_be32(ho->previous_delta, &no->previous_delta);
}

static inline struct ip_vs_sync_buff *
sb_dequeue(struct netns_ipvs *ipvs, struct ipvs_master_sync_state *ms)
{
        struct ip_vs_sync_buff *sb;

        spin_lock_bh(&ipvs->sync_lock);
        if (list_empty(&ms->sync_queue)) {
                sb = NULL;
                __set_current_state(TASK_INTERRUPTIBLE);
        } else {
                sb = list_entry(ms->sync_queue.next, struct ip_vs_sync_buff,
                                list);
                list_del(&sb->list);
                ms->sync_queue_len--;
                if (!ms->sync_queue_len)
                        ms->sync_queue_delay = 0;
        }
        spin_unlock_bh(&ipvs->sync_lock);

        return sb;
}

/*
 * Create a new sync buffer for Version 1 proto.
 */
static inline struct ip_vs_sync_buff *
ip_vs_sync_buff_create(struct netns_ipvs *ipvs, unsigned int len)
{
        struct ip_vs_sync_buff *sb;

        if (!(sb=kmalloc(sizeof(struct ip_vs_sync_buff), GFP_ATOMIC)))
                return NULL;

        len = max_t(unsigned int, len + sizeof(struct ip_vs_sync_mesg),
                    ipvs->mcfg.sync_maxlen);
        sb->mesg = kmalloc(len, GFP_ATOMIC);
        if (!sb->mesg) {
                kfree(sb);
                return NULL;
        }
        sb->mesg->reserved = 0;  /* old nr_conns i.e. must be zero now */
        sb->mesg->version = SYNC_PROTO_VER;
        sb->mesg->syncid = ipvs->mcfg.syncid;
        sb->mesg->size = htons(sizeof(struct ip_vs_sync_mesg));
        sb->mesg->nr_conns = 0;
        sb->mesg->spare = 0;
        sb->head = (unsigned char *)sb->mesg + sizeof(struct ip_vs_sync_mesg);
        sb->end = (unsigned char *)sb->mesg + len;

        sb->firstuse = jiffies;
        return sb;
}

static inline void ip_vs_sync_buff_release(struct ip_vs_sync_buff *sb)
{
        kfree(sb->mesg);
        kfree(sb);
}

static inline void sb_queue_tail(struct netns_ipvs *ipvs,
                                 struct ipvs_master_sync_state *ms)
{
        struct ip_vs_sync_buff *sb = ms->sync_buff;

        spin_lock(&ipvs->sync_lock);
        if (ipvs->sync_state & IP_VS_STATE_MASTER &&
            ms->sync_queue_len < sysctl_sync_qlen_max(ipvs)) {
                if (!ms->sync_queue_len)
                        schedule_delayed_work(&ms->master_wakeup_work,
                                              max(IPVS_SYNC_SEND_DELAY, 1));
                ms->sync_queue_len++;
                list_add_tail(&sb->list, &ms->sync_queue);
                if ((++ms->sync_queue_delay) == IPVS_SYNC_WAKEUP_RATE) {
                        int id = (int)(ms - ipvs->ms);

                        wake_up_process(ipvs->master_tinfo[id].task);
                }
        } else
                ip_vs_sync_buff_release(sb);
        spin_unlock(&ipvs->sync_lock);
}

/*
 *        Get the current sync buffer if it has been created for more
 *        than the specified time or the specified time is zero.
 */
static inline struct ip_vs_sync_buff *
get_curr_sync_buff(struct netns_ipvs *ipvs, struct ipvs_master_sync_state *ms,
                   unsigned long time)
{
        struct ip_vs_sync_buff *sb;

        spin_lock_bh(&ipvs->sync_buff_lock);
        sb = ms->sync_buff;
        if (sb && time_after_eq(jiffies - sb->firstuse, time)) {
                ms->sync_buff = NULL;
                __set_current_state(TASK_RUNNING);
        } else
                sb = NULL;
        spin_unlock_bh(&ipvs->sync_buff_lock);
        return sb;
}

static inline int
select_master_thread_id(struct netns_ipvs *ipvs, struct ip_vs_conn *cp)
{
        return ((long) cp >> (1 + ilog2(sizeof(*cp)))) & ipvs->threads_mask;
}

/*
 * Create a new sync buffer for Version 0 proto.
 */
static inline struct ip_vs_sync_buff *
ip_vs_sync_buff_create_v0(struct netns_ipvs *ipvs, unsigned int len)
{
        struct ip_vs_sync_buff *sb;
        struct ip_vs_sync_mesg_v0 *mesg;

        if (!(sb=kmalloc(sizeof(struct ip_vs_sync_buff), GFP_ATOMIC)))
                return NULL;

        len = max_t(unsigned int, len + sizeof(struct ip_vs_sync_mesg_v0),
                    ipvs->mcfg.sync_maxlen);
        sb->mesg = kmalloc(len, GFP_ATOMIC);
        if (!sb->mesg) {
                kfree(sb);
                return NULL;
        }
        mesg = (struct ip_vs_sync_mesg_v0 *)sb->mesg;
        mesg->nr_conns = 0;
        mesg->syncid = ipvs->mcfg.syncid;
        mesg->size = htons(sizeof(struct ip_vs_sync_mesg_v0));
        sb->head = (unsigned char *)mesg + sizeof(struct ip_vs_sync_mesg_v0);
        sb->end = (unsigned char *)mesg + len;
        sb->firstuse = jiffies;
        return sb;
}

/* Check if connection is controlled by persistence */
static inline bool in_persistence(struct ip_vs_conn *cp)
{
        for (cp = cp->control; cp; cp = cp->control) {
                if (cp->flags & IP_VS_CONN_F_TEMPLATE)
                        return true;
        }
        return false;
}

/* Check if conn should be synced.
 * pkts: conn packets, use sysctl_sync_threshold to avoid packet check
 * - (1) sync_refresh_period: reduce sync rate. Additionally, retry
 *        sync_retries times with period of sync_refresh_period/8
 * - (2) if both sync_refresh_period and sync_period are 0 send sync only
 *        for state changes or only once when pkts matches sync_threshold
 * - (3) templates: rate can be reduced only with sync_refresh_period or
 *        with (2)
 */
static int ip_vs_sync_conn_needed(struct netns_ipvs *ipvs,
                                  struct ip_vs_conn *cp, int pkts)
{
        unsigned long orig = READ_ONCE(cp->sync_endtime);
        unsigned long now = jiffies;
        unsigned long n = (now + cp->timeout) & ~3UL;
        unsigned int sync_refresh_period;
        int sync_period;
        int force;

        /* Check if we sync in current state */
        if (unlikely(cp->flags & IP_VS_CONN_F_TEMPLATE))
                force = 0;
        else if (unlikely(sysctl_sync_persist_mode(ipvs) && in_persistence(cp)))
                return 0;
        else if (likely(cp->protocol == IPPROTO_TCP)) {
                if (!((1 << cp->state) &
                      ((1 << IP_VS_TCP_S_ESTABLISHED) |
                       (1 << IP_VS_TCP_S_FIN_WAIT) |
                       (1 << IP_VS_TCP_S_CLOSE) |
                       (1 << IP_VS_TCP_S_CLOSE_WAIT) |
                       (1 << IP_VS_TCP_S_TIME_WAIT))))
                        return 0;
                force = cp->state != cp->old_state;
                if (force && cp->state != IP_VS_TCP_S_ESTABLISHED)
                        goto set;
        } else if (unlikely(cp->protocol == IPPROTO_SCTP)) {
                if (!((1 << cp->state) &
                      ((1 << IP_VS_SCTP_S_ESTABLISHED) |
                       (1 << IP_VS_SCTP_S_SHUTDOWN_SENT) |
                       (1 << IP_VS_SCTP_S_SHUTDOWN_RECEIVED) |
                       (1 << IP_VS_SCTP_S_SHUTDOWN_ACK_SENT) |
                       (1 << IP_VS_SCTP_S_CLOSED))))
                        return 0;
                force = cp->state != cp->old_state;
                if (force && cp->state != IP_VS_SCTP_S_ESTABLISHED)
                        goto set;
        } else {
                /* UDP or another protocol with single state */
                force = 0;
        }

        sync_refresh_period = sysctl_sync_refresh_period(ipvs);
        if (sync_refresh_period > 0) {
                long diff = n - orig;
                long min_diff = max(cp->timeout >> 1, 10UL * HZ);

                /* Avoid sync if difference is below sync_refresh_period
                 * and below the half timeout.
                 */
                if (abs(diff) < min_t(long, sync_refresh_period, min_diff)) {
                        int retries = orig & 3;

                        if (retries >= sysctl_sync_retries(ipvs))
                                return 0;
                        if (time_before(now, orig - cp->timeout +
                                        (sync_refresh_period >> 3)))
                                return 0;
                        n |= retries + 1;
                }
        }
        sync_period = sysctl_sync_period(ipvs);
        if (sync_period > 0) {
                if (!(cp->flags & IP_VS_CONN_F_TEMPLATE) &&
                    pkts % sync_period != sysctl_sync_threshold(ipvs))
                        return 0;
        } else if (!sync_refresh_period &&
                   pkts != sysctl_sync_threshold(ipvs))
                return 0;

set:
        cp->old_state = cp->state;
        n = cmpxchg(&cp->sync_endtime, orig, n);
        return n == orig || force;
}

/*
 *      Version 0 , could be switched in by sys_ctl.
 *      Add an ip_vs_conn information into the current sync_buff.
 */
static void ip_vs_sync_conn_v0(struct netns_ipvs *ipvs, struct ip_vs_conn *cp,
                               int pkts)
{
        struct ip_vs_sync_mesg_v0 *m;
        struct ip_vs_sync_conn_v0 *s;
        struct ip_vs_sync_buff *buff;
        struct ipvs_master_sync_state *ms;
        int id;
        unsigned int len;

        if (unlikely(cp->af != AF_INET))
                return;
        /* Do not sync ONE PACKET */
        if (cp->flags & IP_VS_CONN_F_ONE_PACKET)
                return;

        if (!ip_vs_sync_conn_needed(ipvs, cp, pkts))
                return;

        spin_lock_bh(&ipvs->sync_buff_lock);
        if (!(ipvs->sync_state & IP_VS_STATE_MASTER)) {
                spin_unlock_bh(&ipvs->sync_buff_lock);
                return;
        }

        id = select_master_thread_id(ipvs, cp);
        ms = &ipvs->ms[id];
        buff = ms->sync_buff;
        len = (cp->flags & IP_VS_CONN_F_SEQ_MASK) ? FULL_CONN_SIZE :
                SIMPLE_CONN_SIZE;
        if (buff) {
                m = (struct ip_vs_sync_mesg_v0 *) buff->mesg;
                /* Send buffer if it is for v1 */
                if (buff->head + len > buff->end || !m->nr_conns) {
                        sb_queue_tail(ipvs, ms);
                        ms->sync_buff = NULL;
                        buff = NULL;
                }
        }
        if (!buff) {
                buff = ip_vs_sync_buff_create_v0(ipvs, len);
                if (!buff) {
                        spin_unlock_bh(&ipvs->sync_buff_lock);
                        pr_err("ip_vs_sync_buff_create failed.\n");
                        return;
                }
                ms->sync_buff = buff;
        }

        m = (struct ip_vs_sync_mesg_v0 *) buff->mesg;
        s = (struct ip_vs_sync_conn_v0 *) buff->head;

        /* copy members */
        s->reserved = 0;
        s->protocol = cp->protocol;
        s->cport = cp->cport;
        s->vport = cp->vport;
        s->dport = cp->dport;
        s->caddr = cp->caddr.ip;
        s->vaddr = cp->vaddr.ip;
        s->daddr = cp->daddr.ip;
        s->flags = htons(cp->flags & ~IP_VS_CONN_F_HASHED);
        s->state = htons(cp->state);
        if (cp->flags & IP_VS_CONN_F_SEQ_MASK) {
                struct ip_vs_sync_conn_options *opt =
                        (struct ip_vs_sync_conn_options *)&s[1];
                memcpy(opt, &cp->sync_conn_opt, sizeof(*opt));
        }

        m->nr_conns++;
        m->size = htons(ntohs(m->size) + len);
        buff->head += len;
        spin_unlock_bh(&ipvs->sync_buff_lock);

        /* synchronize its controller if it has */
        cp = cp->control;
        if (cp) {
                if (cp->flags & IP_VS_CONN_F_TEMPLATE)
                        pkts = atomic_inc_return(&cp->in_pkts);
                else
                        pkts = sysctl_sync_threshold(ipvs);
                ip_vs_sync_conn(ipvs, cp, pkts);
        }
}

/*
 *      Add an ip_vs_conn information into the current sync_buff.
 *      Called by ip_vs_in.
 *      Sending Version 1 messages
 */
void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts)
{
        struct ip_vs_sync_mesg *m;
        union ip_vs_sync_conn *s;
        struct ip_vs_sync_buff *buff;
        struct ipvs_master_sync_state *ms;
        int id;
        __u8 *p;
        unsigned int len, pe_name_len, pad;

        /* Handle old version of the protocol */
        if (sysctl_sync_ver(ipvs) == 0) {
                ip_vs_sync_conn_v0(ipvs, cp, pkts);
                return;
        }
        /* Do not sync ONE PACKET */
        if (cp->flags & IP_VS_CONN_F_ONE_PACKET)
                goto control;
sloop:
        if (!ip_vs_sync_conn_needed(ipvs, cp, pkts))
                goto control;

        /* Sanity checks */
        pe_name_len = 0;
        if (cp->pe_data_len) {
                if (!cp->pe_data || !cp->dest) {
                        IP_VS_ERR_RL("SYNC, connection pe_data invalid\n");
                        return;
                }
                pe_name_len = strnlen(cp->pe->name, IP_VS_PENAME_MAXLEN);
        }

        spin_lock_bh(&ipvs->sync_buff_lock);
        if (!(ipvs->sync_state & IP_VS_STATE_MASTER)) {
                spin_unlock_bh(&ipvs->sync_buff_lock);
                return;
        }

        id = select_master_thread_id(ipvs, cp);
        ms = &ipvs->ms[id];

#ifdef CONFIG_IP_VS_IPV6
        if (cp->af == AF_INET6)
                len = sizeof(struct ip_vs_sync_v6);
        else
#endif
                len = sizeof(struct ip_vs_sync_v4);

        if (cp->flags & IP_VS_CONN_F_SEQ_MASK)
                len += sizeof(struct ip_vs_sync_conn_options) + 2;

        if (cp->pe_data_len)
                len += cp->pe_data_len + 2;        /* + Param hdr field */
        if (pe_name_len)
                len += pe_name_len + 2;

        /* check if there is a space for this one  */
        pad = 0;
        buff = ms->sync_buff;
        if (buff) {
                m = buff->mesg;
                pad = (4 - (size_t) buff->head) & 3;
                /* Send buffer if it is for v0 */
                if (buff->head + len + pad > buff->end || m->reserved) {
                        sb_queue_tail(ipvs, ms);
                        ms->sync_buff = NULL;
                        buff = NULL;
                        pad = 0;
                }
        }

        if (!buff) {
                buff = ip_vs_sync_buff_create(ipvs, len);
                if (!buff) {
                        spin_unlock_bh(&ipvs->sync_buff_lock);
                        pr_err("ip_vs_sync_buff_create failed.\n");
                        return;
                }
                ms->sync_buff = buff;
                m = buff->mesg;
        }

        p = buff->head;
        buff->head += pad + len;
        m->size = htons(ntohs(m->size) + pad + len);
        /* Add ev. padding from prev. sync_conn */
        while (pad--)
                *(p++) = 0;

        s = (union ip_vs_sync_conn *)p;

        /* Set message type  & copy members */
        s->v4.type = (cp->af == AF_INET6 ? STYPE_F_INET6 : 0);
        s->v4.ver_size = htons(len & SVER_MASK);        /* Version 0 */
        s->v4.flags = htonl(cp->flags & ~IP_VS_CONN_F_HASHED);
        s->v4.state = htons(cp->state);
        s->v4.protocol = cp->protocol;
        s->v4.cport = cp->cport;
        s->v4.vport = cp->vport;
        s->v4.dport = cp->dport;
        s->v4.fwmark = htonl(cp->fwmark);
        s->v4.timeout = htonl(cp->timeout / HZ);
        m->nr_conns++;

#ifdef CONFIG_IP_VS_IPV6
        if (cp->af == AF_INET6) {
                p += sizeof(struct ip_vs_sync_v6);
                s->v6.caddr = cp->caddr.in6;
                s->v6.vaddr = cp->vaddr.in6;
                s->v6.daddr = cp->daddr.in6;
        } else
#endif
        {
                p += sizeof(struct ip_vs_sync_v4);        /* options ptr */
                s->v4.caddr = cp->caddr.ip;
                s->v4.vaddr = cp->vaddr.ip;
                s->v4.daddr = cp->daddr.ip;
        }
        if (cp->flags & IP_VS_CONN_F_SEQ_MASK) {
                *(p++) = IPVS_OPT_SEQ_DATA;
                *(p++) = sizeof(struct ip_vs_sync_conn_options);
                hton_seq((struct ip_vs_seq *)p, &cp->in_seq);
                p += sizeof(struct ip_vs_seq);
                hton_seq((struct ip_vs_seq *)p, &cp->out_seq);
                p += sizeof(struct ip_vs_seq);
        }
        /* Handle pe data */
        if (cp->pe_data_len && cp->pe_data) {
                *(p++) = IPVS_OPT_PE_DATA;
                *(p++) = cp->pe_data_len;
                memcpy(p, cp->pe_data, cp->pe_data_len);
                p += cp->pe_data_len;
                if (pe_name_len) {
                        /* Add PE_NAME */
                        *(p++) = IPVS_OPT_PE_NAME;
                        *(p++) = pe_name_len;
                        memcpy(p, cp->pe->name, pe_name_len);
                        p += pe_name_len;
                }
        }

        spin_unlock_bh(&ipvs->sync_buff_lock);

control:
        /* synchronize its controller if it has */
        cp = cp->control;
        if (!cp)
                return;
        if (cp->flags & IP_VS_CONN_F_TEMPLATE)
                pkts = atomic_inc_return(&cp->in_pkts);
        else
                pkts = sysctl_sync_threshold(ipvs);
        goto sloop;
}

/*
 *  fill_param used by version 1
 */
static inline int
ip_vs_conn_fill_param_sync(struct netns_ipvs *ipvs, int af, union ip_vs_sync_conn *sc,
                           struct ip_vs_conn_param *p,
                           __u8 *pe_data, unsigned int pe_data_len,
                           __u8 *pe_name, unsigned int pe_name_len)
{
#ifdef CONFIG_IP_VS_IPV6
        if (af == AF_INET6)
                ip_vs_conn_fill_param(ipvs, af, sc->v6.protocol,
                                      (const union nf_inet_addr *)&sc->v6.caddr,
                                      sc->v6.cport,
                                      (const union nf_inet_addr *)&sc->v6.vaddr,
                                      sc->v6.vport, p);
        else
#endif
                ip_vs_conn_fill_param(ipvs, af, sc->v4.protocol,
                                      (const union nf_inet_addr *)&sc->v4.caddr,
                                      sc->v4.cport,
                                      (const union nf_inet_addr *)&sc->v4.vaddr,
                                      sc->v4.vport, p);
        /* Handle pe data */
        if (pe_data_len) {
                if (pe_name_len) {
                        char buff[IP_VS_PENAME_MAXLEN+1];

                        memcpy(buff, pe_name, pe_name_len);
                        buff[pe_name_len]=0;
                        p->pe = __ip_vs_pe_getbyname(buff);
                        if (!p->pe) {
                                IP_VS_DBG(3, "BACKUP, no %s engine found/loaded\n",
                                             buff);
                                return 1;
                        }
                } else {
                        IP_VS_ERR_RL("BACKUP, Invalid PE parameters\n");
                        return 1;
                }

                p->pe_data = kmemdup(pe_data, pe_data_len, GFP_ATOMIC);
                if (!p->pe_data) {
                        module_put(p->pe->module);
                        return -ENOMEM;
                }
                p->pe_data_len = pe_data_len;
        }
        return 0;
}

/*
 *  Connection Add / Update.
 *  Common for version 0 and 1 reception of backup sync_conns.
 *  Param: ...
 *         timeout is in sec.
 */
static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *param,
                            unsigned int flags, unsigned int state,
                            unsigned int protocol, unsigned int type,
                            const union nf_inet_addr *daddr, __be16 dport,
                            unsigned long timeout, __u32 fwmark,
                            struct ip_vs_sync_conn_options *opt)
{
        struct ip_vs_dest *dest;
        struct ip_vs_conn *cp;

        if (!(flags & IP_VS_CONN_F_TEMPLATE)) {
                cp = ip_vs_conn_in_get(param);
                if (cp && ((cp->dport != dport) ||
                           !ip_vs_addr_equal(cp->daf, &cp->daddr, daddr))) {
                        if (!(flags & IP_VS_CONN_F_INACTIVE)) {
                                ip_vs_conn_expire_now(cp);
                                __ip_vs_conn_put(cp);
                                cp = NULL;
                        } else {
                                /* This is the expiration message for the
                                 * connection that was already replaced, so we
                                 * just ignore it.
                                 */
                                __ip_vs_conn_put(cp);
                                kfree(param->pe_data);
                                return;
                        }
                }
        } else {
                cp = ip_vs_ct_in_get(param);
        }

        if (cp) {
                /* Free pe_data */
                kfree(param->pe_data);

                dest = cp->dest;
                spin_lock_bh(&cp->lock);
                if ((cp->flags ^ flags) & IP_VS_CONN_F_INACTIVE &&
                    !(flags & IP_VS_CONN_F_TEMPLATE) && dest) {
                        if (flags & IP_VS_CONN_F_INACTIVE) {
                                atomic_dec(&dest->activeconns);
                                atomic_inc(&dest->inactconns);
                        } else {
                                atomic_inc(&dest->activeconns);
                                atomic_dec(&dest->inactconns);
                        }
                }
                flags &= IP_VS_CONN_F_BACKUP_UPD_MASK;
                flags |= cp->flags & ~IP_VS_CONN_F_BACKUP_UPD_MASK;
                cp->flags = flags;
                spin_unlock_bh(&cp->lock);
                if (!dest)
                        ip_vs_try_bind_dest(cp);
        } else {
                /*
                 * Find the appropriate destination for the connection.
                 * If it is not found the connection will remain unbound
                 * but still handled.
                 */
                rcu_read_lock();
                /* This function is only invoked by the synchronization
                 * code. We do not currently support heterogeneous pools
                 * with synchronization, so we can make the assumption that
                 * the svc_af is the same as the dest_af
                 */
                dest = ip_vs_find_dest(ipvs, type, type, daddr, dport,
                                       param->vaddr, param->vport, protocol,
                                       fwmark, flags);

                cp = ip_vs_conn_new(param, type, daddr, dport, flags, dest,
                                    fwmark);
                rcu_read_unlock();
                if (!cp) {
                        kfree(param->pe_data);
                        IP_VS_DBG(2, "BACKUP, add new conn. failed\n");
                        return;
                }
                if (!(flags & IP_VS_CONN_F_TEMPLATE))
                        kfree(param->pe_data);
        }

        if (opt) {
                cp->in_seq = opt->in_seq;
                cp->out_seq = opt->out_seq;
        }
        atomic_set(&cp->in_pkts, sysctl_sync_threshold(ipvs));
        cp->state = state;
        cp->old_state = cp->state;
        /*
         * For Ver 0 messages style
         *  - Not possible to recover the right timeout for templates
         *  - can not find the right fwmark
         *    virtual service. If needed, we can do it for
         *    non-fwmark persistent services.
         * Ver 1 messages style.
         *  - No problem.
         */
        if (timeout) {
                if (timeout > MAX_SCHEDULE_TIMEOUT / HZ)
                        timeout = MAX_SCHEDULE_TIMEOUT / HZ;
                cp->timeout = timeout*HZ;
        } else {
                struct ip_vs_proto_data *pd;

                pd = ip_vs_proto_data_get(ipvs, protocol);
                if (!(flags & IP_VS_CONN_F_TEMPLATE) && pd && pd->timeout_table)
                        cp->timeout = pd->timeout_table[state];
                else
                        cp->timeout = (3*60*HZ);
        }
        ip_vs_conn_put(cp);
}

/*
 *  Process received multicast message for Version 0
 */
static void ip_vs_process_message_v0(struct netns_ipvs *ipvs, const char *buffer,
                                     const size_t buflen)
{
        struct ip_vs_sync_mesg_v0 *m = (struct ip_vs_sync_mesg_v0 *)buffer;
        struct ip_vs_sync_conn_v0 *s;
        struct ip_vs_sync_conn_options *opt;
        struct ip_vs_protocol *pp;
        struct ip_vs_conn_param param;
        char *p;
        int i;

        p = (char *)buffer + sizeof(struct ip_vs_sync_mesg_v0);
        for (i=0; i<m->nr_conns; i++) {
                unsigned int flags, state;

                if (p + SIMPLE_CONN_SIZE > buffer+buflen) {
                        IP_VS_ERR_RL("BACKUP v0, bogus conn\n");
                        return;
                }
                s = (struct ip_vs_sync_conn_v0 *) p;
                flags = ntohs(s->flags) | IP_VS_CONN_F_SYNC;
                flags &= ~IP_VS_CONN_F_HASHED;
                if (flags & IP_VS_CONN_F_SEQ_MASK) {
                        opt = (struct ip_vs_sync_conn_options *)&s[1];
                        p += FULL_CONN_SIZE;
                        if (p > buffer+buflen) {
                                IP_VS_ERR_RL("BACKUP v0, Dropping buffer bogus conn options\n");
                                return;
                        }
                } else {
                        opt = NULL;
                        p += SIMPLE_CONN_SIZE;
                }

                state = ntohs(s->state);
                if (!(flags & IP_VS_CONN_F_TEMPLATE)) {
                        pp = ip_vs_proto_get(s->protocol);
                        if (!pp) {
                                IP_VS_DBG(2, "BACKUP v0, Unsupported protocol %u\n",
                                        s->protocol);
                                continue;
                        }
                        if (state >= pp->num_states) {
                                IP_VS_DBG(2, "BACKUP v0, Invalid %s state %u\n",
                                        pp->name, state);
                                continue;
                        }
                } else {
                        if (state >= IP_VS_CTPL_S_LAST)
                                IP_VS_DBG(7, "BACKUP v0, Invalid tpl state %u\n",
                                          state);
                }

                ip_vs_conn_fill_param(ipvs, AF_INET, s->protocol,
                                      (const union nf_inet_addr *)&s->caddr,
                                      s->cport,
                                      (const union nf_inet_addr *)&s->vaddr,
                                      s->vport, &param);

                /* Send timeout as Zero */
                ip_vs_proc_conn(ipvs, &param, flags, state, s->protocol, AF_INET,
                                (union nf_inet_addr *)&s->daddr, s->dport,
                                0, 0, opt);
        }
}

/*
 * Handle options
 */
static inline int ip_vs_proc_seqopt(__u8 *p, unsigned int plen,
                                    __u32 *opt_flags,
                                    struct ip_vs_sync_conn_options *opt)
{
        struct ip_vs_sync_conn_options *topt;

        topt = (struct ip_vs_sync_conn_options *)p;

        if (plen != sizeof(struct ip_vs_sync_conn_options)) {
                IP_VS_DBG(2, "BACKUP, bogus conn options length\n");
                return -EINVAL;
        }
        if (*opt_flags & IPVS_OPT_F_SEQ_DATA) {
                IP_VS_DBG(2, "BACKUP, conn options found twice\n");
                return -EINVAL;
        }
        ntoh_seq(&topt->in_seq, &opt->in_seq);
        ntoh_seq(&topt->out_seq, &opt->out_seq);
        *opt_flags |= IPVS_OPT_F_SEQ_DATA;
        return 0;
}

static int ip_vs_proc_str(__u8 *p, unsigned int plen, unsigned int *data_len,
                          __u8 **data, unsigned int maxlen,
                          __u32 *opt_flags, __u32 flag)
{
        if (plen > maxlen) {
                IP_VS_DBG(2, "BACKUP, bogus par.data len > %d\n", maxlen);
                return -EINVAL;
        }
        if (*opt_flags & flag) {
                IP_VS_DBG(2, "BACKUP, Par.data found twice 0x%x\n", flag);
                return -EINVAL;
        }
        *data_len = plen;
        *data = p;
        *opt_flags |= flag;
        return 0;
}
/*
 *   Process a Version 1 sync. connection
 */
static inline int ip_vs_proc_sync_conn(struct netns_ipvs *ipvs, __u8 *p, __u8 *msg_end)
{
        struct ip_vs_sync_conn_options opt;
        union  ip_vs_sync_conn *s;
        struct ip_vs_protocol *pp;
        struct ip_vs_conn_param param;
        __u32 flags;
        unsigned int af, state, pe_data_len=0, pe_name_len=0;
        __u8 *pe_data=NULL, *pe_name=NULL;
        __u32 opt_flags=0;
        int retc=0;

        s = (union ip_vs_sync_conn *) p;

        if (s->v6.type & STYPE_F_INET6) {
#ifdef CONFIG_IP_VS_IPV6
                af = AF_INET6;
                p += sizeof(struct ip_vs_sync_v6);
#else
                IP_VS_DBG(3,"BACKUP, IPv6 msg received, and IPVS is not compiled for IPv6\n");
                retc = 10;
                goto out;
#endif
        } else if (!s->v4.type) {
                af = AF_INET;
                p += sizeof(struct ip_vs_sync_v4);
        } else {
                return -10;
        }
        if (p > msg_end)
                return -20;

        /* Process optional params check Type & Len. */
        while (p < msg_end) {
                int ptype;
                int plen;

                if (p+2 > msg_end)
                        return -30;
                ptype = *(p++);
                plen  = *(p++);

                if (!plen || ((p + plen) > msg_end))
                        return -40;
                /* Handle seq option  p = param data */
                switch (ptype & ~IPVS_OPT_F_PARAM) {
                case IPVS_OPT_SEQ_DATA:
                        if (ip_vs_proc_seqopt(p, plen, &opt_flags, &opt))
                                return -50;
                        break;

                case IPVS_OPT_PE_DATA:
                        if (ip_vs_proc_str(p, plen, &pe_data_len, &pe_data,
                                           IP_VS_PEDATA_MAXLEN, &opt_flags,
                                           IPVS_OPT_F_PE_DATA))
                                return -60;
                        break;

                case IPVS_OPT_PE_NAME:
                        if (ip_vs_proc_str(p, plen,&pe_name_len, &pe_name,
                                           IP_VS_PENAME_MAXLEN, &opt_flags,
                                           IPVS_OPT_F_PE_NAME))
                                return -70;
                        break;

                default:
                        /* Param data mandatory ? */
                        if (!(ptype & IPVS_OPT_F_PARAM)) {
                                IP_VS_DBG(3, "BACKUP, Unknown mandatory param %d found\n",
                                          ptype & ~IPVS_OPT_F_PARAM);
                                retc = 20;
                                goto out;
                        }
                }
                p += plen;  /* Next option */
        }

        /* Get flags and Mask off unsupported */
        flags  = ntohl(s->v4.flags) & IP_VS_CONN_F_BACKUP_MASK;
        flags |= IP_VS_CONN_F_SYNC;
        state = ntohs(s->v4.state);

        if (!(flags & IP_VS_CONN_F_TEMPLATE)) {
                pp = ip_vs_proto_get(s->v4.protocol);
                if (!pp) {
                        IP_VS_DBG(3,"BACKUP, Unsupported protocol %u\n",
                                s->v4.protocol);
                        retc = 30;
                        goto out;
                }
                if (state >= pp->num_states) {
                        IP_VS_DBG(3, "BACKUP, Invalid %s state %u\n",
                                pp->name, state);
                        retc = 40;
                        goto out;
                }
        } else {
                if (state >= IP_VS_CTPL_S_LAST)
                        IP_VS_DBG(7, "BACKUP, Invalid tpl state %u\n",
                                  state);
        }
        if (ip_vs_conn_fill_param_sync(ipvs, af, s, &param, pe_data,
                                       pe_data_len, pe_name, pe_name_len)) {
                retc = 50;
                goto out;
        }
        /* If only IPv4, just silent skip IPv6 */
        if (af == AF_INET)
                ip_vs_proc_conn(ipvs, &param, flags, state, s->v4.protocol, af,
                                (union nf_inet_addr *)&s->v4.daddr, s->v4.dport,
                                ntohl(s->v4.timeout), ntohl(s->v4.fwmark),
                                (opt_flags & IPVS_OPT_F_SEQ_DATA ? &opt : NULL)
                                );
#ifdef CONFIG_IP_VS_IPV6
        else
                ip_vs_proc_conn(ipvs, &param, flags, state, s->v6.protocol, af,
                                (union nf_inet_addr *)&s->v6.daddr, s->v6.dport,
                                ntohl(s->v6.timeout), ntohl(s->v6.fwmark),
                                (opt_flags & IPVS_OPT_F_SEQ_DATA ? &opt : NULL)
                                );
#endif
        ip_vs_pe_put(param.pe);
        return 0;
        /* Error exit */
out:
        IP_VS_DBG(2, "BACKUP, Single msg dropped err:%d\n", retc);
        return retc;

}
/*
 *      Process received multicast message and create the corresponding
 *      ip_vs_conn entries.
 *      Handles Version 0 & 1
 */
static void ip_vs_process_message(struct netns_ipvs *ipvs, __u8 *buffer,
                                  const size_t buflen)
{
        struct ip_vs_sync_mesg *m2 = (struct ip_vs_sync_mesg *)buffer;
        __u8 *p, *msg_end;
        int i, nr_conns;

        if (buflen < sizeof(struct ip_vs_sync_mesg_v0)) {
                IP_VS_DBG(2, "BACKUP, message header too short\n");
                return;
        }

        if (buflen != ntohs(m2->size)) {
                IP_VS_DBG(2, "BACKUP, bogus message size\n");
                return;
        }
        /* SyncID sanity check */
        if (ipvs->bcfg.syncid != 0 && m2->syncid != ipvs->bcfg.syncid) {
                IP_VS_DBG(7, "BACKUP, Ignoring syncid = %d\n", m2->syncid);
                return;
        }
        /* Handle version 1  message */
        if ((m2->version == SYNC_PROTO_VER) && (m2->reserved == 0)
            && (m2->spare == 0)) {

                msg_end = buffer + sizeof(struct ip_vs_sync_mesg);
                nr_conns = m2->nr_conns;

                for (i=0; i<nr_conns; i++) {
                        union ip_vs_sync_conn *s;
                        unsigned int size;
                        int retc;

                        p = msg_end;
                        if (p + sizeof(s->v4) > buffer+buflen) {
                                IP_VS_ERR_RL("BACKUP, Dropping buffer, too small\n");
                                return;
                        }
                        s = (union ip_vs_sync_conn *)p;
                        size = ntohs(s->v4.ver_size) & SVER_MASK;
                        msg_end = p + size;
                        /* Basic sanity checks */
                        if (msg_end  > buffer+buflen) {
                                IP_VS_ERR_RL("BACKUP, Dropping buffer, msg > buffer\n");
                                return;
                        }
                        if (ntohs(s->v4.ver_size) >> SVER_SHIFT) {
                                IP_VS_ERR_RL("BACKUP, Dropping buffer, Unknown version %d\n",
                                              ntohs(s->v4.ver_size) >> SVER_SHIFT);
                                return;
                        }
                        /* Process a single sync_conn */
                        retc = ip_vs_proc_sync_conn(ipvs, p, msg_end);
                        if (retc < 0) {
                                IP_VS_ERR_RL("BACKUP, Dropping buffer, Err: %d in decoding\n",
                                             retc);
                                return;
                        }
                        /* Make sure we have 32 bit alignment */
                        msg_end = p + ((size + 3) & ~3);
                }
        } else {
                /* Old type of message */
                ip_vs_process_message_v0(ipvs, buffer, buflen);
                return;
        }
}


/*
 *      Setup sndbuf (mode=1) or rcvbuf (mode=0)
 */
static void set_sock_size(struct sock *sk, int mode, int val)
{
        /* setsockopt(sock, SOL_SOCKET, SO_SNDBUF, &val, sizeof(val)); */
        /* setsockopt(sock, SOL_SOCKET, SO_RCVBUF, &val, sizeof(val)); */
        lock_sock(sk);
        if (mode) {
                val = clamp_t(int, val, (SOCK_MIN_SNDBUF + 1) / 2,
                              READ_ONCE(sysctl_wmem_max));
                sk->sk_sndbuf = val * 2;
                sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
        } else {
                val = clamp_t(int, val, (SOCK_MIN_RCVBUF + 1) / 2,
                              READ_ONCE(sysctl_rmem_max));
                sk->sk_rcvbuf = val * 2;
                sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
        }
        release_sock(sk);
}

/*
 *      Setup loopback of outgoing multicasts on a sending socket
 */
static void set_mcast_loop(struct sock *sk, u_char loop)
{
        /* setsockopt(sock, SOL_IP, IP_MULTICAST_LOOP, &loop, sizeof(loop)); */
        inet_assign_bit(MC_LOOP, sk, loop);
#ifdef CONFIG_IP_VS_IPV6
        if (READ_ONCE(sk->sk_family) == AF_INET6) {
                /* IPV6_MULTICAST_LOOP */
                inet6_assign_bit(MC6_LOOP, sk, loop);
        }
#endif
}

/*
 *      Specify TTL for outgoing multicasts on a sending socket
 */
static void set_mcast_ttl(struct sock *sk, u_char ttl)
{
        struct inet_sock *inet = inet_sk(sk);

        /* setsockopt(sock, SOL_IP, IP_MULTICAST_TTL, &ttl, sizeof(ttl)); */
        lock_sock(sk);
        WRITE_ONCE(inet->mc_ttl, ttl);
#ifdef CONFIG_IP_VS_IPV6
        if (sk->sk_family == AF_INET6) {
                struct ipv6_pinfo *np = inet6_sk(sk);

                /* IPV6_MULTICAST_HOPS */
                WRITE_ONCE(np->mcast_hops, ttl);
        }
#endif
        release_sock(sk);
}

/* Control fragmentation of messages */
static void set_mcast_pmtudisc(struct sock *sk, int val)
{
        struct inet_sock *inet = inet_sk(sk);

        /* setsockopt(sock, SOL_IP, IP_MTU_DISCOVER, &val, sizeof(val)); */
        lock_sock(sk);
        WRITE_ONCE(inet->pmtudisc, val);
#ifdef CONFIG_IP_VS_IPV6
        if (sk->sk_family == AF_INET6) {
                struct ipv6_pinfo *np = inet6_sk(sk);

                /* IPV6_MTU_DISCOVER */
                WRITE_ONCE(np->pmtudisc, val);
        }
#endif
        release_sock(sk);
}

/*
 *      Specifiy default interface for outgoing multicasts
 */
static int set_mcast_if(struct sock *sk, struct net_device *dev)
{
        struct inet_sock *inet = inet_sk(sk);

        if (sk->sk_bound_dev_if && dev->ifindex != sk->sk_bound_dev_if)
                return -EINVAL;

        lock_sock(sk);
        inet->mc_index = dev->ifindex;
        /*  inet->mc_addr  = 0; */
#ifdef CONFIG_IP_VS_IPV6
        if (sk->sk_family == AF_INET6) {
                struct ipv6_pinfo *np = inet6_sk(sk);

                /* IPV6_MULTICAST_IF */
                WRITE_ONCE(np->mcast_oif, dev->ifindex);
        }
#endif
        release_sock(sk);

        return 0;
}


/*
 *      Join a multicast group.
 *      the group is specified by a class D multicast address 224.0.0.0/8
 *      in the in_addr structure passed in as a parameter.
 */
static int
join_mcast_group(struct sock *sk, struct in_addr *addr, struct net_device *dev)
{
        struct ip_mreqn mreq;
        int ret;

        memset(&mreq, 0, sizeof(mreq));
        memcpy(&mreq.imr_multiaddr, addr, sizeof(struct in_addr));

        if (sk->sk_bound_dev_if && dev->ifindex != sk->sk_bound_dev_if)
                return -EINVAL;

        mreq.imr_ifindex = dev->ifindex;

        lock_sock(sk);
        ret = ip_mc_join_group(sk, &mreq);
        release_sock(sk);

        return ret;
}

#ifdef CONFIG_IP_VS_IPV6
static int join_mcast_group6(struct sock *sk, struct in6_addr *addr,
                             struct net_device *dev)
{
        int ret;

        if (sk->sk_bound_dev_if && dev->ifindex != sk->sk_bound_dev_if)
                return -EINVAL;

        lock_sock(sk);
        ret = ipv6_sock_mc_join(sk, dev->ifindex, addr);
        release_sock(sk);

        return ret;
}
#endif

static int bind_mcastif_addr(struct socket *sock, struct net_device *dev)
{
        __be32 addr;
        struct sockaddr_in sin;

        addr = inet_select_addr(dev, 0, RT_SCOPE_UNIVERSE);
        if (!addr)
                pr_err("You probably need to specify IP address on "
                       "multicast interface.\n");

        IP_VS_DBG(7, "binding socket with (%s) %pI4\n",
                  dev->name, &addr);

        /* Now bind the socket with the address of multicast interface */
        sin.sin_family             = AF_INET;
        sin.sin_addr.s_addr  = addr;
        sin.sin_port         = 0;

        return kernel_bind(sock, (struct sockaddr *)&sin, sizeof(sin));
}

static void get_mcast_sockaddr(union ipvs_sockaddr *sa, int *salen,
                               struct ipvs_sync_daemon_cfg *c, int id)
{
        if (AF_INET6 == c->mcast_af) {
                sa->in6 = (struct sockaddr_in6) {
                        .sin6_family = AF_INET6,
                        .sin6_port = htons(c->mcast_port + id),
                };
                sa->in6.sin6_addr = c->mcast_group.in6;
                *salen = sizeof(sa->in6);
        } else {
                sa->in = (struct sockaddr_in) {
                        .sin_family = AF_INET,
                        .sin_port = htons(c->mcast_port + id),
                };
                sa->in.sin_addr = c->mcast_group.in;
                *salen = sizeof(sa->in);
        }
}

/*
 *      Set up sending multicast socket over UDP
 */
static int make_send_sock(struct netns_ipvs *ipvs, int id,
                          struct net_device *dev, struct socket **sock_ret)
{
        /* multicast addr */
        union ipvs_sockaddr mcast_addr;
        struct socket *sock;
        int result, salen;

        /* First create a socket */
        result = sock_create_kern(ipvs->net, ipvs->mcfg.mcast_af, SOCK_DGRAM,
                                  IPPROTO_UDP, &sock);
        if (result < 0) {
                pr_err("Error during creation of socket; terminating\n");
                goto error;
        }
        *sock_ret = sock;
        result = set_mcast_if(sock->sk, dev);
        if (result < 0) {
                pr_err("Error setting outbound mcast interface\n");
                goto error;
        }

        set_mcast_loop(sock->sk, 0);
        set_mcast_ttl(sock->sk, ipvs->mcfg.mcast_ttl);
        /* Allow fragmentation if MTU changes */
        set_mcast_pmtudisc(sock->sk, IP_PMTUDISC_DONT);
        result = sysctl_sync_sock_size(ipvs);
        if (result > 0)
                set_sock_size(sock->sk, 1, result);

        if (AF_INET == ipvs->mcfg.mcast_af)
                result = bind_mcastif_addr(sock, dev);
        else
                result = 0;
        if (result < 0) {
                pr_err("Error binding address of the mcast interface\n");
                goto error;
        }

        get_mcast_sockaddr(&mcast_addr, &salen, &ipvs->mcfg, id);
        result = kernel_connect(sock, (struct sockaddr *)&mcast_addr,
                                salen, 0);
        if (result < 0) {
                pr_err("Error connecting to the multicast addr\n");
                goto error;
        }

        return 0;

error:
        return result;
}


/*
 *      Set up receiving multicast socket over UDP
 */
static int make_receive_sock(struct netns_ipvs *ipvs, int id,
                             struct net_device *dev, struct socket **sock_ret)
{
        /* multicast addr */
        union ipvs_sockaddr mcast_addr;
        struct socket *sock;
        int result, salen;

        /* First create a socket */
        result = sock_create_kern(ipvs->net, ipvs->bcfg.mcast_af, SOCK_DGRAM,
                                  IPPROTO_UDP, &sock);
        if (result < 0) {
                pr_err("Error during creation of socket; terminating\n");
                goto error;
        }
        *sock_ret = sock;
        /* it is equivalent to the REUSEADDR option in user-space */
        sock->sk->sk_reuse = SK_CAN_REUSE;
        result = sysctl_sync_sock_size(ipvs);
        if (result > 0)
                set_sock_size(sock->sk, 0, result);

        get_mcast_sockaddr(&mcast_addr, &salen, &ipvs->bcfg, id);
        sock->sk->sk_bound_dev_if = dev->ifindex;
        result = kernel_bind(sock, (struct sockaddr *)&mcast_addr, salen);
        if (result < 0) {
                pr_err("Error binding to the multicast addr\n");
                goto error;
        }

        /* join the multicast group */
#ifdef CONFIG_IP_VS_IPV6
        if (ipvs->bcfg.mcast_af == AF_INET6)
                result = join_mcast_group6(sock->sk, &mcast_addr.in6.sin6_addr,
                                           dev);
        else
#endif
                result = join_mcast_group(sock->sk, &mcast_addr.in.sin_addr,
                                          dev);
        if (result < 0) {
                pr_err("Error joining to the multicast group\n");
                goto error;
        }

        return 0;

error:
        return result;
}


static int
ip_vs_send_async(struct socket *sock, const char *buffer, const size_t length)
{
        struct msghdr        msg = {.msg_flags = MSG_DONTWAIT|MSG_NOSIGNAL};
        struct kvec        iov;
        int                len;

        iov.iov_base     = (void *)buffer;
        iov.iov_len      = length;

        len = kernel_sendmsg(sock, &msg, &iov, 1, (size_t)(length));

        return len;
}

static int
ip_vs_send_sync_msg(struct socket *sock, struct ip_vs_sync_mesg *msg)
{
        int msize;
        int ret;

        msize = ntohs(msg->size);

        ret = ip_vs_send_async(sock, (char *)msg, msize);
        if (ret >= 0 || ret == -EAGAIN)
                return ret;
        pr_err("ip_vs_send_async error %d\n", ret);
        return 0;
}

static int
ip_vs_receive(struct socket *sock, char *buffer, const size_t buflen)
{
        struct msghdr                msg = {NULL,};
        struct kvec                iov = {buffer, buflen};
        int                        len;

        /* Receive a packet */
        iov_iter_kvec(&msg.msg_iter, ITER_DEST, &iov, 1, buflen);
        len = sock_recvmsg(sock, &msg, MSG_DONTWAIT);
        if (len < 0)
                return len;

        return len;
}

/* Wakeup the master thread for sending */
static void master_wakeup_work_handler(struct work_struct *work)
{
        struct ipvs_master_sync_state *ms =
                container_of(work, struct ipvs_master_sync_state,
                             master_wakeup_work.work);
        struct netns_ipvs *ipvs = ms->ipvs;

        spin_lock_bh(&ipvs->sync_lock);
        if (ms->sync_queue_len &&
            ms->sync_queue_delay < IPVS_SYNC_WAKEUP_RATE) {
                int id = (int)(ms - ipvs->ms);

                ms->sync_queue_delay = IPVS_SYNC_WAKEUP_RATE;
                wake_up_process(ipvs->master_tinfo[id].task);
        }
        spin_unlock_bh(&ipvs->sync_lock);
}

/* Get next buffer to send */
static inline struct ip_vs_sync_buff *
next_sync_buff(struct netns_ipvs *ipvs, struct ipvs_master_sync_state *ms)
{
        struct ip_vs_sync_buff *sb;

        sb = sb_dequeue(ipvs, ms);
        if (sb)
                return sb;
        /* Do not delay entries in buffer for more than 2 seconds */
        return get_curr_sync_buff(ipvs, ms, IPVS_SYNC_FLUSH_TIME);
}

static int sync_thread_master(void *data)
{
        struct ip_vs_sync_thread_data *tinfo = data;
        struct netns_ipvs *ipvs = tinfo->ipvs;
        struct ipvs_master_sync_state *ms = &ipvs->ms[tinfo->id];
        struct sock *sk = tinfo->sock->sk;
        struct ip_vs_sync_buff *sb;

        pr_info("sync thread started: state = MASTER, mcast_ifn = %s, "
                "syncid = %d, id = %d\n",
                ipvs->mcfg.mcast_ifn, ipvs->mcfg.syncid, tinfo->id);

        for (;;) {
                sb = next_sync_buff(ipvs, ms);
                if (unlikely(kthread_should_stop()))
                        break;
                if (!sb) {
                        schedule_timeout(IPVS_SYNC_CHECK_PERIOD);
                        continue;
                }
                while (ip_vs_send_sync_msg(tinfo->sock, sb->mesg) < 0) {
                        /* (Ab)use interruptible sleep to avoid increasing
                         * the load avg.
                         */
                        __wait_event_interruptible(*sk_sleep(sk),
                                                   sock_writeable(sk) ||
                                                   kthread_should_stop());
                        if (unlikely(kthread_should_stop()))
                                goto done;
                }
                ip_vs_sync_buff_release(sb);
        }

done:
        __set_current_state(TASK_RUNNING);
        if (sb)
                ip_vs_sync_buff_release(sb);

        /* clean up the sync_buff queue */
        while ((sb = sb_dequeue(ipvs, ms)))
                ip_vs_sync_buff_release(sb);
        __set_current_state(TASK_RUNNING);

        /* clean up the current sync_buff */
        sb = get_curr_sync_buff(ipvs, ms, 0);
        if (sb)
                ip_vs_sync_buff_release(sb);

        return 0;
}


static int sync_thread_backup(void *data)
{
        struct ip_vs_sync_thread_data *tinfo = data;
        struct netns_ipvs *ipvs = tinfo->ipvs;
        struct sock *sk = tinfo->sock->sk;
        struct udp_sock *up = udp_sk(sk);
        int len;

        pr_info("sync thread started: state = BACKUP, mcast_ifn = %s, "
                "syncid = %d, id = %d\n",
                ipvs->bcfg.mcast_ifn, ipvs->bcfg.syncid, tinfo->id);

        while (!kthread_should_stop()) {
                wait_event_interruptible(*sk_sleep(sk),
                                         !skb_queue_empty_lockless(&sk->sk_receive_queue) ||
                                         !skb_queue_empty_lockless(&up->reader_queue) ||
                                         kthread_should_stop());

                /* do we have data now? */
                while (!skb_queue_empty_lockless(&sk->sk_receive_queue) ||
                       !skb_queue_empty_lockless(&up->reader_queue)) {
                        len = ip_vs_receive(tinfo->sock, tinfo->buf,
                                        ipvs->bcfg.sync_maxlen);
                        if (len <= 0) {
                                if (len != -EAGAIN)
                                        pr_err("receiving message error\n");
                                break;
                        }

                        ip_vs_process_message(ipvs, tinfo->buf, len);
                }
        }

        return 0;
}


int start_sync_thread(struct netns_ipvs *ipvs, struct ipvs_sync_daemon_cfg *c,
                      int state)
{
        struct ip_vs_sync_thread_data *ti = NULL, *tinfo;
        struct task_struct *task;
        struct net_device *dev;
        char *name;
        int (*threadfn)(void *data);
        int id = 0, count, hlen;
        int result = -ENOMEM;
        u16 mtu, min_mtu;

        IP_VS_DBG(7, "%s(): pid %d\n", __func__, task_pid_nr(current));
        IP_VS_DBG(7, "Each ip_vs_sync_conn entry needs %zd bytes\n",
                  sizeof(struct ip_vs_sync_conn_v0));

        /* increase the module use count */
        if (!ip_vs_use_count_inc())
                return -ENOPROTOOPT;

        /* Do not hold one mutex and then to block on another */
        for (;;) {
                rtnl_lock();
                if (mutex_trylock(&ipvs->sync_mutex))
                        break;
                rtnl_unlock();
                mutex_lock(&ipvs->sync_mutex);
                if (rtnl_trylock())
                        break;
                mutex_unlock(&ipvs->sync_mutex);
        }

        if (!ipvs->sync_state) {
                count = clamp(sysctl_sync_ports(ipvs), 1, IPVS_SYNC_PORTS_MAX);
                ipvs->threads_mask = count - 1;
        } else
                count = ipvs->threads_mask + 1;

        if (c->mcast_af == AF_UNSPEC) {
                c->mcast_af = AF_INET;
                c->mcast_group.ip = cpu_to_be32(IP_VS_SYNC_GROUP);
        }
        if (!c->mcast_port)
                c->mcast_port = IP_VS_SYNC_PORT;
        if (!c->mcast_ttl)
                c->mcast_ttl = 1;

        dev = __dev_get_by_name(ipvs->net, c->mcast_ifn);
        if (!dev) {
                pr_err("Unknown mcast interface: %s\n", c->mcast_ifn);
                result = -ENODEV;
                goto out_early;
        }
        hlen = (AF_INET6 == c->mcast_af) ?
               sizeof(struct ipv6hdr) + sizeof(struct udphdr) :
               sizeof(struct iphdr) + sizeof(struct udphdr);
        mtu = (state == IP_VS_STATE_BACKUP) ?
                  clamp(dev->mtu, 1500U, 65535U) : 1500U;
        min_mtu = (state == IP_VS_STATE_BACKUP) ? 1024 : 1;

        if (c->sync_maxlen)
                c->sync_maxlen = clamp_t(unsigned int,
                                         c->sync_maxlen, min_mtu,
                                         65535 - hlen);
        else
                c->sync_maxlen = mtu - hlen;

        if (state == IP_VS_STATE_MASTER) {
                result = -EEXIST;
                if (ipvs->ms)
                        goto out_early;

                ipvs->mcfg = *c;
                name = "ipvs-m:%d:%d";
                threadfn = sync_thread_master;
        } else if (state == IP_VS_STATE_BACKUP) {
                result = -EEXIST;
                if (ipvs->backup_tinfo)
                        goto out_early;

                ipvs->bcfg = *c;
                name = "ipvs-b:%d:%d";
                threadfn = sync_thread_backup;
        } else {
                result = -EINVAL;
                goto out_early;
        }

        if (state == IP_VS_STATE_MASTER) {
                struct ipvs_master_sync_state *ms;

                result = -ENOMEM;
                ipvs->ms = kcalloc(count, sizeof(ipvs->ms[0]), GFP_KERNEL);
                if (!ipvs->ms)
                        goto out;
                ms = ipvs->ms;
                for (id = 0; id < count; id++, ms++) {
                        INIT_LIST_HEAD(&ms->sync_queue);
                        ms->sync_queue_len = 0;
                        ms->sync_queue_delay = 0;
                        INIT_DELAYED_WORK(&ms->master_wakeup_work,
                                          master_wakeup_work_handler);
                        ms->ipvs = ipvs;
                }
        }
        result = -ENOMEM;
        ti = kcalloc(count, sizeof(struct ip_vs_sync_thread_data),
                     GFP_KERNEL);
        if (!ti)
                goto out;

        for (id = 0; id < count; id++) {
                tinfo = &ti[id];
                tinfo->ipvs = ipvs;
                if (state == IP_VS_STATE_BACKUP) {
                        result = -ENOMEM;
                        tinfo->buf = kmalloc(ipvs->bcfg.sync_maxlen,
                                             GFP_KERNEL);
                        if (!tinfo->buf)
                                goto out;
                }
                tinfo->id = id;
                if (state == IP_VS_STATE_MASTER)
                        result = make_send_sock(ipvs, id, dev, &tinfo->sock);
                else
                        result = make_receive_sock(ipvs, id, dev, &tinfo->sock);
                if (result < 0)
                        goto out;

                task = kthread_run(threadfn, tinfo, name, ipvs->gen, id);
                if (IS_ERR(task)) {
                        result = PTR_ERR(task);
                        goto out;
                }
                tinfo->task = task;
        }

        /* mark as active */

        if (state == IP_VS_STATE_MASTER)
                ipvs->master_tinfo = ti;
        else
                ipvs->backup_tinfo = ti;
        spin_lock_bh(&ipvs->sync_buff_lock);
        ipvs->sync_state |= state;
        spin_unlock_bh(&ipvs->sync_buff_lock);

        mutex_unlock(&ipvs->sync_mutex);
        rtnl_unlock();

        return 0;

out:
        /* We do not need RTNL lock anymore, release it here so that
         * sock_release below can use rtnl_lock to leave the mcast group.
         */
        rtnl_unlock();
        id = min(id, count - 1);
        if (ti) {
                for (tinfo = ti + id; tinfo >= ti; tinfo--) {
                        if (tinfo->task)
                                kthread_stop(tinfo->task);
                }
        }
        if (!(ipvs->sync_state & IP_VS_STATE_MASTER)) {
                kfree(ipvs->ms);
                ipvs->ms = NULL;
        }
        mutex_unlock(&ipvs->sync_mutex);

        /* No more mutexes, release socks */
        if (ti) {
                for (tinfo = ti + id; tinfo >= ti; tinfo--) {
                        if (tinfo->sock)
                                sock_release(tinfo->sock);
                        kfree(tinfo->buf);
                }
                kfree(ti);
        }

        /* decrease the module use count */
        ip_vs_use_count_dec();
        return result;

out_early:
        mutex_unlock(&ipvs->sync_mutex);
        rtnl_unlock();

        /* decrease the module use count */
        ip_vs_use_count_dec();
        return result;
}


int stop_sync_thread(struct netns_ipvs *ipvs, int state)
{
        struct ip_vs_sync_thread_data *ti, *tinfo;
        int id;
        int retc = -EINVAL;

        IP_VS_DBG(7, "%s(): pid %d\n", __func__, task_pid_nr(current));

        mutex_lock(&ipvs->sync_mutex);
        if (state == IP_VS_STATE_MASTER) {
                retc = -ESRCH;
                if (!ipvs->ms)
                        goto err;
                ti = ipvs->master_tinfo;

                /*
                 * The lock synchronizes with sb_queue_tail(), so that we don't
                 * add sync buffers to the queue, when we are already in
                 * progress of stopping the master sync daemon.
                 */

                spin_lock_bh(&ipvs->sync_buff_lock);
                spin_lock(&ipvs->sync_lock);
                ipvs->sync_state &= ~IP_VS_STATE_MASTER;
                spin_unlock(&ipvs->sync_lock);
                spin_unlock_bh(&ipvs->sync_buff_lock);

                retc = 0;
                for (id = ipvs->threads_mask; id >= 0; id--) {
                        struct ipvs_master_sync_state *ms = &ipvs->ms[id];
                        int ret;

                        tinfo = &ti[id];
                        pr_info("stopping master sync thread %d ...\n",
                                task_pid_nr(tinfo->task));
                        cancel_delayed_work_sync(&ms->master_wakeup_work);
                        ret = kthread_stop(tinfo->task);
                        if (retc >= 0)
                                retc = ret;
                }
                kfree(ipvs->ms);
                ipvs->ms = NULL;
                ipvs->master_tinfo = NULL;
        } else if (state == IP_VS_STATE_BACKUP) {
                retc = -ESRCH;
                if (!ipvs->backup_tinfo)
                        goto err;
                ti = ipvs->backup_tinfo;

                ipvs->sync_state &= ~IP_VS_STATE_BACKUP;
                retc = 0;
                for (id = ipvs->threads_mask; id >= 0; id--) {
                        int ret;

                        tinfo = &ti[id];
                        pr_info("stopping backup sync thread %d ...\n",
                                task_pid_nr(tinfo->task));
                        ret = kthread_stop(tinfo->task);
                        if (retc >= 0)
                                retc = ret;
                }
                ipvs->backup_tinfo = NULL;
        } else {
                goto err;
        }
        id = ipvs->threads_mask;
        mutex_unlock(&ipvs->sync_mutex);

        /* No more mutexes, release socks */
        for (tinfo = ti + id; tinfo >= ti; tinfo--) {
                if (tinfo->sock)
                        sock_release(tinfo->sock);
                kfree(tinfo->buf);
        }
        kfree(ti);

        /* decrease the module use count */
        ip_vs_use_count_dec();
        return retc;

err:
        mutex_unlock(&ipvs->sync_mutex);
        return retc;
}

/*
 * Initialize data struct for each netns
 */
int __net_init ip_vs_sync_net_init(struct netns_ipvs *ipvs)
{
        __mutex_init(&ipvs->sync_mutex, "ipvs->sync_mutex", &__ipvs_sync_key);
        spin_lock_init(&ipvs->sync_lock);
        spin_lock_init(&ipvs->sync_buff_lock);
        return 0;
}

void ip_vs_sync_net_cleanup(struct netns_ipvs *ipvs)
{
        int retc;

        retc = stop_sync_thread(ipvs, IP_VS_STATE_MASTER);
        if (retc && retc != -ESRCH)
                pr_err("Failed to stop Master Daemon\n");

        retc = stop_sync_thread(ipvs, IP_VS_STATE_BACKUP);
        if (retc && retc != -ESRCH)
                pr_err("Failed to stop Backup Daemon\n");
}

























































   33 




   33 

   33 








































































































































































   33 



   33 



























































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
// SPDX-License-Identifier: GPL-2.0-or-later
/* RxRPC virtual connection handler, common bits.
 *
 * Copyright (C) 2007, 2016 Red Hat, Inc. All Rights Reserved.
 * Written by David Howells (dhowells@redhat.com)
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/net.h>
#include <linux/skbuff.h>
#include "ar-internal.h"

/*
 * Time till a connection expires after last use (in seconds).
 */
unsigned int __read_mostly rxrpc_connection_expiry = 10 * 60;
unsigned int __read_mostly rxrpc_closed_conn_expiry = 10;

static void rxrpc_clean_up_connection(struct work_struct *work);
static void rxrpc_set_service_reap_timer(struct rxrpc_net *rxnet,
                                         unsigned long reap_at);

void rxrpc_poke_conn(struct rxrpc_connection *conn, enum rxrpc_conn_trace why)
{
        struct rxrpc_local *local = conn->local;
        bool busy;

        if (WARN_ON_ONCE(!local))
                return;

        spin_lock_bh(&local->lock);
        busy = !list_empty(&conn->attend_link);
        if (!busy) {
                rxrpc_get_connection(conn, why);
                list_add_tail(&conn->attend_link, &local->conn_attend_q);
        }
        spin_unlock_bh(&local->lock);
        rxrpc_wake_up_io_thread(local);
}

static void rxrpc_connection_timer(struct timer_list *timer)
{
        struct rxrpc_connection *conn =
                container_of(timer, struct rxrpc_connection, timer);

        rxrpc_poke_conn(conn, rxrpc_conn_get_poke_timer);
}

/*
 * allocate a new connection
 */
struct rxrpc_connection *rxrpc_alloc_connection(struct rxrpc_net *rxnet,
                                                gfp_t gfp)
{
        struct rxrpc_connection *conn;

        _enter("");

        conn = kzalloc(sizeof(struct rxrpc_connection), gfp);
        if (conn) {
                INIT_LIST_HEAD(&conn->cache_link);
                timer_setup(&conn->timer, &rxrpc_connection_timer, 0);
                INIT_WORK(&conn->processor, rxrpc_process_connection);
                INIT_WORK(&conn->destructor, rxrpc_clean_up_connection);
                INIT_LIST_HEAD(&conn->proc_link);
                INIT_LIST_HEAD(&conn->link);
                mutex_init(&conn->security_lock);
                mutex_init(&conn->tx_data_alloc_lock);
                skb_queue_head_init(&conn->rx_queue);
                conn->rxnet = rxnet;
                conn->security = &rxrpc_no_security;
                spin_lock_init(&conn->state_lock);
                conn->debug_id = atomic_inc_return(&rxrpc_debug_id);
                conn->idle_timestamp = jiffies;
        }

        _leave(" = %p{%d}", conn, conn ? conn->debug_id : 0);
        return conn;
}

/*
 * Look up a connection in the cache by protocol parameters.
 *
 * If successful, a pointer to the connection is returned, but no ref is taken.
 * NULL is returned if there is no match.
 *
 * When searching for a service call, if we find a peer but no connection, we
 * return that through *_peer in case we need to create a new service call.
 *
 * The caller must be holding the RCU read lock.
 */
struct rxrpc_connection *rxrpc_find_client_connection_rcu(struct rxrpc_local *local,
                                                          struct sockaddr_rxrpc *srx,
                                                          struct sk_buff *skb)
{
        struct rxrpc_connection *conn;
        struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
        struct rxrpc_peer *peer;

        _enter(",%x", sp->hdr.cid & RXRPC_CIDMASK);

        /* Look up client connections by connection ID alone as their
         * IDs are unique for this machine.
         */
        conn = idr_find(&local->conn_ids, sp->hdr.cid >> RXRPC_CIDSHIFT);
        if (!conn || refcount_read(&conn->ref) == 0) {
                _debug("no conn");
                goto not_found;
        }

        if (conn->proto.epoch != sp->hdr.epoch ||
            conn->local != local)
                goto not_found;

        peer = conn->peer;
        switch (srx->transport.family) {
        case AF_INET:
                if (peer->srx.transport.sin.sin_port !=
                    srx->transport.sin.sin_port)
                        goto not_found;
                break;
#ifdef CONFIG_AF_RXRPC_IPV6
        case AF_INET6:
                if (peer->srx.transport.sin6.sin6_port !=
                    srx->transport.sin6.sin6_port)
                        goto not_found;
                break;
#endif
        default:
                BUG();
        }

        _leave(" = %p", conn);
        return conn;

not_found:
        _leave(" = NULL");
        return NULL;
}

/*
 * Disconnect a call and clear any channel it occupies when that call
 * terminates.  The caller must hold the channel_lock and must release the
 * call's ref on the connection.
 */
void __rxrpc_disconnect_call(struct rxrpc_connection *conn,
                             struct rxrpc_call *call)
{
        struct rxrpc_channel *chan =
                &conn->channels[call->cid & RXRPC_CHANNELMASK];

        _enter("%d,%x", conn->debug_id, call->cid);

        if (chan->call == call) {
                /* Save the result of the call so that we can repeat it if necessary
                 * through the channel, whilst disposing of the actual call record.
                 */
                trace_rxrpc_disconnect_call(call);
                switch (call->completion) {
                case RXRPC_CALL_SUCCEEDED:
                        chan->last_seq = call->rx_highest_seq;
                        chan->last_type = RXRPC_PACKET_TYPE_ACK;
                        break;
                case RXRPC_CALL_LOCALLY_ABORTED:
                        chan->last_abort = call->abort_code;
                        chan->last_type = RXRPC_PACKET_TYPE_ABORT;
                        break;
                default:
                        chan->last_abort = RX_CALL_DEAD;
                        chan->last_type = RXRPC_PACKET_TYPE_ABORT;
                        break;
                }

                chan->last_call = chan->call_id;
                chan->call_id = chan->call_counter;
                chan->call = NULL;
        }

        _leave("");
}

/*
 * Disconnect a call and clear any channel it occupies when that call
 * terminates.
 */
void rxrpc_disconnect_call(struct rxrpc_call *call)
{
        struct rxrpc_connection *conn = call->conn;

        set_bit(RXRPC_CALL_DISCONNECTED, &call->flags);
        rxrpc_see_call(call, rxrpc_call_see_disconnected);

        call->peer->cong_ssthresh = call->cong_ssthresh;

        if (!hlist_unhashed(&call->error_link)) {
                spin_lock(&call->peer->lock);
                hlist_del_init(&call->error_link);
                spin_unlock(&call->peer->lock);
        }

        if (rxrpc_is_client_call(call)) {
                rxrpc_disconnect_client_call(call->bundle, call);
        } else {
                __rxrpc_disconnect_call(conn, call);
                conn->idle_timestamp = jiffies;
                if (atomic_dec_and_test(&conn->active))
                        rxrpc_set_service_reap_timer(conn->rxnet,
                                                     jiffies + rxrpc_connection_expiry * HZ);
        }

        rxrpc_put_call(call, rxrpc_call_put_io_thread);
}

/*
 * Queue a connection's work processor, getting a ref to pass to the work
 * queue.
 */
void rxrpc_queue_conn(struct rxrpc_connection *conn, enum rxrpc_conn_trace why)
{
        if (atomic_read(&conn->active) >= 0 &&
            rxrpc_queue_work(&conn->processor))
                rxrpc_see_connection(conn, why);
}

/*
 * Note the re-emergence of a connection.
 */
void rxrpc_see_connection(struct rxrpc_connection *conn,
                          enum rxrpc_conn_trace why)
{
        if (conn) {
                int r = refcount_read(&conn->ref);

                trace_rxrpc_conn(conn->debug_id, r, why);
        }
}

/*
 * Get a ref on a connection.
 */
struct rxrpc_connection *rxrpc_get_connection(struct rxrpc_connection *conn,
                                              enum rxrpc_conn_trace why)
{
        int r;

        __refcount_inc(&conn->ref, &r);
        trace_rxrpc_conn(conn->debug_id, r + 1, why);
        return conn;
}

/*
 * Try to get a ref on a connection.
 */
struct rxrpc_connection *
rxrpc_get_connection_maybe(struct rxrpc_connection *conn,
                           enum rxrpc_conn_trace why)
{
        int r;

        if (conn) {
                if (__refcount_inc_not_zero(&conn->ref, &r))
                        trace_rxrpc_conn(conn->debug_id, r + 1, why);
                else
                        conn = NULL;
        }
        return conn;
}

/*
 * Set the service connection reap timer.
 */
static void rxrpc_set_service_reap_timer(struct rxrpc_net *rxnet,
                                         unsigned long reap_at)
{
        if (rxnet->live)
                timer_reduce(&rxnet->service_conn_reap_timer, reap_at);
}

/*
 * destroy a virtual connection
 */
static void rxrpc_rcu_free_connection(struct rcu_head *rcu)
{
        struct rxrpc_connection *conn =
                container_of(rcu, struct rxrpc_connection, rcu);
        struct rxrpc_net *rxnet = conn->rxnet;

        _enter("{%d,u=%d}", conn->debug_id, refcount_read(&conn->ref));

        trace_rxrpc_conn(conn->debug_id, refcount_read(&conn->ref),
                         rxrpc_conn_free);
        kfree(conn);

        if (atomic_dec_and_test(&rxnet->nr_conns))
                wake_up_var(&rxnet->nr_conns);
}

/*
 * Clean up a dead connection.
 */
static void rxrpc_clean_up_connection(struct work_struct *work)
{
        struct rxrpc_connection *conn =
                container_of(work, struct rxrpc_connection, destructor);
        struct rxrpc_net *rxnet = conn->rxnet;

        ASSERT(!conn->channels[0].call &&
               !conn->channels[1].call &&
               !conn->channels[2].call &&
               !conn->channels[3].call);
        ASSERT(list_empty(&conn->cache_link));

        del_timer_sync(&conn->timer);
        cancel_work_sync(&conn->processor); /* Processing may restart the timer */
        del_timer_sync(&conn->timer);

        write_lock(&rxnet->conn_lock);
        list_del_init(&conn->proc_link);
        write_unlock(&rxnet->conn_lock);

        rxrpc_purge_queue(&conn->rx_queue);

        rxrpc_kill_client_conn(conn);

        conn->security->clear(conn);
        key_put(conn->key);
        rxrpc_put_bundle(conn->bundle, rxrpc_bundle_put_conn);
        rxrpc_put_peer(conn->peer, rxrpc_peer_put_conn);
        rxrpc_put_local(conn->local, rxrpc_local_put_kill_conn);

        /* Drain the Rx queue.  Note that even though we've unpublished, an
         * incoming packet could still be being added to our Rx queue, so we
         * will need to drain it again in the RCU cleanup handler.
         */
        rxrpc_purge_queue(&conn->rx_queue);

        if (conn->tx_data_alloc.va)
                __page_frag_cache_drain(virt_to_page(conn->tx_data_alloc.va),
                                        conn->tx_data_alloc.pagecnt_bias);
        call_rcu(&conn->rcu, rxrpc_rcu_free_connection);
}

/*
 * Drop a ref on a connection.
 */
void rxrpc_put_connection(struct rxrpc_connection *conn,
                          enum rxrpc_conn_trace why)
{
        unsigned int debug_id;
        bool dead;
        int r;

        if (!conn)
                return;

        debug_id = conn->debug_id;
        dead = __refcount_dec_and_test(&conn->ref, &r);
        trace_rxrpc_conn(debug_id, r - 1, why);
        if (dead) {
                del_timer(&conn->timer);
                cancel_work(&conn->processor);

                if (in_softirq() || work_busy(&conn->processor) ||
                    timer_pending(&conn->timer))
                        /* Can't use the rxrpc workqueue as we need to cancel/flush
                         * something that may be running/waiting there.
                         */
                        schedule_work(&conn->destructor);
                else
                        rxrpc_clean_up_connection(&conn->destructor);
        }
}

/*
 * reap dead service connections
 */
void rxrpc_service_connection_reaper(struct work_struct *work)
{
        struct rxrpc_connection *conn, *_p;
        struct rxrpc_net *rxnet =
                container_of(work, struct rxrpc_net, service_conn_reaper);
        unsigned long expire_at, earliest, idle_timestamp, now;
        int active;

        LIST_HEAD(graveyard);

        _enter("");

        now = jiffies;
        earliest = now + MAX_JIFFY_OFFSET;

        write_lock(&rxnet->conn_lock);
        list_for_each_entry_safe(conn, _p, &rxnet->service_conns, link) {
                ASSERTCMP(atomic_read(&conn->active), >=, 0);
                if (likely(atomic_read(&conn->active) > 0))
                        continue;
                if (conn->state == RXRPC_CONN_SERVICE_PREALLOC)
                        continue;

                if (rxnet->live && !conn->local->dead) {
                        idle_timestamp = READ_ONCE(conn->idle_timestamp);
                        expire_at = idle_timestamp + rxrpc_connection_expiry * HZ;
                        if (conn->local->service_closed)
                                expire_at = idle_timestamp + rxrpc_closed_conn_expiry * HZ;

                        _debug("reap CONN %d { a=%d,t=%ld }",
                               conn->debug_id, atomic_read(&conn->active),
                               (long)expire_at - (long)now);

                        if (time_before(now, expire_at)) {
                                if (time_before(expire_at, earliest))
                                        earliest = expire_at;
                                continue;
                        }
                }

                /* The activity count sits at 0 whilst the conn is unused on
                 * the list; we reduce that to -1 to make the conn unavailable.
                 */
                active = 0;
                if (!atomic_try_cmpxchg(&conn->active, &active, -1))
                        continue;
                rxrpc_see_connection(conn, rxrpc_conn_see_reap_service);

                if (rxrpc_conn_is_client(conn))
                        BUG();
                else
                        rxrpc_unpublish_service_conn(conn);

                list_move_tail(&conn->link, &graveyard);
        }
        write_unlock(&rxnet->conn_lock);

        if (earliest != now + MAX_JIFFY_OFFSET) {
                _debug("reschedule reaper %ld", (long)earliest - (long)now);
                ASSERT(time_after(earliest, now));
                rxrpc_set_service_reap_timer(rxnet, earliest);
        }

        while (!list_empty(&graveyard)) {
                conn = list_entry(graveyard.next, struct rxrpc_connection,
                                  link);
                list_del_init(&conn->link);

                ASSERTCMP(atomic_read(&conn->active), ==, -1);
                rxrpc_put_connection(conn, rxrpc_conn_put_service_reaped);
        }

        _leave("");
}

/*
 * preemptively destroy all the service connection records rather than
 * waiting for them to time out
 */
void rxrpc_destroy_all_connections(struct rxrpc_net *rxnet)
{
        struct rxrpc_connection *conn, *_p;
        bool leak = false;

        _enter("");

        atomic_dec(&rxnet->nr_conns);

        del_timer_sync(&rxnet->service_conn_reap_timer);
        rxrpc_queue_work(&rxnet->service_conn_reaper);
        flush_workqueue(rxrpc_workqueue);

        write_lock(&rxnet->conn_lock);
        list_for_each_entry_safe(conn, _p, &rxnet->service_conns, link) {
                pr_err("AF_RXRPC: Leaked conn %p {%d}\n",
                       conn, refcount_read(&conn->ref));
                leak = true;
        }
        write_unlock(&rxnet->conn_lock);
        BUG_ON(leak);

        ASSERT(list_empty(&rxnet->conn_proc_list));

        /* We need to wait for the connections to be destroyed by RCU as they
         * pin things that we still need to get rid of.
         */
        wait_var_event(&rxnet->nr_conns, !atomic_read(&rxnet->nr_conns));
        _leave("");
}















































































































































































































    1 


    9 



    4 



   15 






    1 


   14 




   14 










   16 


































   16 






   16 



   16 









































































































    2 


    5 

    1 


    4 




    1 





    2 

    4 
    2 







    8 





















































   14 










    1 



    1 






    1 



   12 











   12 









   12 

































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/cls_route.c        ROUTE4 classifier.
 *
 * Authors:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <net/dst.h>
#include <net/route.h>
#include <net/netlink.h>
#include <net/act_api.h>
#include <net/pkt_cls.h>
#include <net/tc_wrapper.h>

/*
 * 1. For now we assume that route tags < 256.
 *    It allows to use direct table lookups, instead of hash tables.
 * 2. For now we assume that "from TAG" and "fromdev DEV" statements
 *    are mutually  exclusive.
 * 3. "to TAG from ANY" has higher priority, than "to ANY from XXX"
 */
struct route4_fastmap {
        struct route4_filter                *filter;
        u32                                id;
        int                                iif;
};

struct route4_head {
        struct route4_fastmap                fastmap[16];
        struct route4_bucket __rcu        *table[256 + 1];
        struct rcu_head                        rcu;
};

struct route4_bucket {
        /* 16 FROM buckets + 16 IIF buckets + 1 wildcard bucket */
        struct route4_filter __rcu        *ht[16 + 16 + 1];
        struct rcu_head                        rcu;
};

struct route4_filter {
        struct route4_filter __rcu        *next;
        u32                        id;
        int                        iif;

        struct tcf_result        res;
        struct tcf_exts                exts;
        u32                        handle;
        struct route4_bucket        *bkt;
        struct tcf_proto        *tp;
        struct rcu_work                rwork;
};

#define ROUTE4_FAILURE ((struct route4_filter *)(-1L))

static inline int route4_fastmap_hash(u32 id, int iif)
{
        return id & 0xF;
}

static DEFINE_SPINLOCK(fastmap_lock);
static void
route4_reset_fastmap(struct route4_head *head)
{
        spin_lock_bh(&fastmap_lock);
        memset(head->fastmap, 0, sizeof(head->fastmap));
        spin_unlock_bh(&fastmap_lock);
}

static void
route4_set_fastmap(struct route4_head *head, u32 id, int iif,
                   struct route4_filter *f)
{
        int h = route4_fastmap_hash(id, iif);

        /* fastmap updates must look atomic to aling id, iff, filter */
        spin_lock_bh(&fastmap_lock);
        head->fastmap[h].id = id;
        head->fastmap[h].iif = iif;
        head->fastmap[h].filter = f;
        spin_unlock_bh(&fastmap_lock);
}

static inline int route4_hash_to(u32 id)
{
        return id & 0xFF;
}

static inline int route4_hash_from(u32 id)
{
        return (id >> 16) & 0xF;
}

static inline int route4_hash_iif(int iif)
{
        return 16 + ((iif >> 16) & 0xF);
}

static inline int route4_hash_wild(void)
{
        return 32;
}

#define ROUTE4_APPLY_RESULT()                                        \
{                                                                \
        *res = f->res;                                                \
        if (tcf_exts_has_actions(&f->exts)) {                        \
                int r = tcf_exts_exec(skb, &f->exts, res);        \
                if (r < 0) {                                        \
                        dont_cache = 1;                                \
                        continue;                                \
                }                                                \
                return r;                                        \
        } else if (!dont_cache)                                        \
                route4_set_fastmap(head, id, iif, f);                \
        return 0;                                                \
}

TC_INDIRECT_SCOPE int route4_classify(struct sk_buff *skb,
                                      const struct tcf_proto *tp,
                                      struct tcf_result *res)
{
        struct route4_head *head = rcu_dereference_bh(tp->root);
        struct dst_entry *dst;
        struct route4_bucket *b;
        struct route4_filter *f;
        u32 id, h;
        int iif, dont_cache = 0;

        dst = skb_dst(skb);
        if (!dst)
                goto failure;

        id = dst->tclassid;

        iif = inet_iif(skb);

        h = route4_fastmap_hash(id, iif);

        spin_lock(&fastmap_lock);
        if (id == head->fastmap[h].id &&
            iif == head->fastmap[h].iif &&
            (f = head->fastmap[h].filter) != NULL) {
                if (f == ROUTE4_FAILURE) {
                        spin_unlock(&fastmap_lock);
                        goto failure;
                }

                *res = f->res;
                spin_unlock(&fastmap_lock);
                return 0;
        }
        spin_unlock(&fastmap_lock);

        h = route4_hash_to(id);

restart:
        b = rcu_dereference_bh(head->table[h]);
        if (b) {
                for (f = rcu_dereference_bh(b->ht[route4_hash_from(id)]);
                     f;
                     f = rcu_dereference_bh(f->next))
                        if (f->id == id)
                                ROUTE4_APPLY_RESULT();

                for (f = rcu_dereference_bh(b->ht[route4_hash_iif(iif)]);
                     f;
                     f = rcu_dereference_bh(f->next))
                        if (f->iif == iif)
                                ROUTE4_APPLY_RESULT();

                for (f = rcu_dereference_bh(b->ht[route4_hash_wild()]);
                     f;
                     f = rcu_dereference_bh(f->next))
                        ROUTE4_APPLY_RESULT();
        }
        if (h < 256) {
                h = 256;
                id &= ~0xFFFF;
                goto restart;
        }

        if (!dont_cache)
                route4_set_fastmap(head, id, iif, ROUTE4_FAILURE);
failure:
        return -1;
}

static inline u32 to_hash(u32 id)
{
        u32 h = id & 0xFF;

        if (id & 0x8000)
                h += 256;
        return h;
}

static inline u32 from_hash(u32 id)
{
        id &= 0xFFFF;
        if (id == 0xFFFF)
                return 32;
        if (!(id & 0x8000)) {
                if (id > 255)
                        return 256;
                return id & 0xF;
        }
        return 16 + (id & 0xF);
}

static void *route4_get(struct tcf_proto *tp, u32 handle)
{
        struct route4_head *head = rtnl_dereference(tp->root);
        struct route4_bucket *b;
        struct route4_filter *f;
        unsigned int h1, h2;

        h1 = to_hash(handle);
        if (h1 > 256)
                return NULL;

        h2 = from_hash(handle >> 16);
        if (h2 > 32)
                return NULL;

        b = rtnl_dereference(head->table[h1]);
        if (b) {
                for (f = rtnl_dereference(b->ht[h2]);
                     f;
                     f = rtnl_dereference(f->next))
                        if (f->handle == handle)
                                return f;
        }
        return NULL;
}

static int route4_init(struct tcf_proto *tp)
{
        struct route4_head *head;

        head = kzalloc(sizeof(struct route4_head), GFP_KERNEL);
        if (head == NULL)
                return -ENOBUFS;

        rcu_assign_pointer(tp->root, head);
        return 0;
}

static void __route4_delete_filter(struct route4_filter *f)
{
        tcf_exts_destroy(&f->exts);
        tcf_exts_put_net(&f->exts);
        kfree(f);
}

static void route4_delete_filter_work(struct work_struct *work)
{
        struct route4_filter *f = container_of(to_rcu_work(work),
                                               struct route4_filter,
                                               rwork);
        rtnl_lock();
        __route4_delete_filter(f);
        rtnl_unlock();
}

static void route4_queue_work(struct route4_filter *f)
{
        tcf_queue_work(&f->rwork, route4_delete_filter_work);
}

static void route4_destroy(struct tcf_proto *tp, bool rtnl_held,
                           struct netlink_ext_ack *extack)
{
        struct route4_head *head = rtnl_dereference(tp->root);
        int h1, h2;

        if (head == NULL)
                return;

        for (h1 = 0; h1 <= 256; h1++) {
                struct route4_bucket *b;

                b = rtnl_dereference(head->table[h1]);
                if (b) {
                        for (h2 = 0; h2 <= 32; h2++) {
                                struct route4_filter *f;

                                while ((f = rtnl_dereference(b->ht[h2])) != NULL) {
                                        struct route4_filter *next;

                                        next = rtnl_dereference(f->next);
                                        RCU_INIT_POINTER(b->ht[h2], next);
                                        tcf_unbind_filter(tp, &f->res);
                                        if (tcf_exts_get_net(&f->exts))
                                                route4_queue_work(f);
                                        else
                                                __route4_delete_filter(f);
                                }
                        }
                        RCU_INIT_POINTER(head->table[h1], NULL);
                        kfree_rcu(b, rcu);
                }
        }
        kfree_rcu(head, rcu);
}

static int route4_delete(struct tcf_proto *tp, void *arg, bool *last,
                         bool rtnl_held, struct netlink_ext_ack *extack)
{
        struct route4_head *head = rtnl_dereference(tp->root);
        struct route4_filter *f = arg;
        struct route4_filter __rcu **fp;
        struct route4_filter *nf;
        struct route4_bucket *b;
        unsigned int h = 0;
        int i, h1;

        if (!head || !f)
                return -EINVAL;

        h = f->handle;
        b = f->bkt;

        fp = &b->ht[from_hash(h >> 16)];
        for (nf = rtnl_dereference(*fp); nf;
             fp = &nf->next, nf = rtnl_dereference(*fp)) {
                if (nf == f) {
                        /* unlink it */
                        RCU_INIT_POINTER(*fp, rtnl_dereference(f->next));

                        /* Remove any fastmap lookups that might ref filter
                         * notice we unlink'd the filter so we can't get it
                         * back in the fastmap.
                         */
                        route4_reset_fastmap(head);

                        /* Delete it */
                        tcf_unbind_filter(tp, &f->res);
                        tcf_exts_get_net(&f->exts);
                        tcf_queue_work(&f->rwork, route4_delete_filter_work);

                        /* Strip RTNL protected tree */
                        for (i = 0; i <= 32; i++) {
                                struct route4_filter *rt;

                                rt = rtnl_dereference(b->ht[i]);
                                if (rt)
                                        goto out;
                        }

                        /* OK, session has no flows */
                        RCU_INIT_POINTER(head->table[to_hash(h)], NULL);
                        kfree_rcu(b, rcu);
                        break;
                }
        }

out:
        *last = true;
        for (h1 = 0; h1 <= 256; h1++) {
                if (rcu_access_pointer(head->table[h1])) {
                        *last = false;
                        break;
                }
        }

        return 0;
}

static const struct nla_policy route4_policy[TCA_ROUTE4_MAX + 1] = {
        [TCA_ROUTE4_CLASSID]        = { .type = NLA_U32 },
        [TCA_ROUTE4_TO]                = NLA_POLICY_MAX(NLA_U32, 0xFF),
        [TCA_ROUTE4_FROM]        = NLA_POLICY_MAX(NLA_U32, 0xFF),
        [TCA_ROUTE4_IIF]        = NLA_POLICY_MAX(NLA_U32, 0x7FFF),
};

static int route4_set_parms(struct net *net, struct tcf_proto *tp,
                            unsigned long base, struct route4_filter *f,
                            u32 handle, struct route4_head *head,
                            struct nlattr **tb, struct nlattr *est, int new,
                            u32 flags, struct netlink_ext_ack *extack)
{
        u32 id = 0, to = 0, nhandle = 0x8000;
        struct route4_filter *fp;
        unsigned int h1;
        struct route4_bucket *b;
        int err;

        err = tcf_exts_validate(net, tp, tb, est, &f->exts, flags, extack);
        if (err < 0)
                return err;

        if (tb[TCA_ROUTE4_TO]) {
                if (new && handle & 0x8000) {
                        NL_SET_ERR_MSG(extack, "Invalid handle");
                        return -EINVAL;
                }
                to = nla_get_u32(tb[TCA_ROUTE4_TO]);
                nhandle = to;
        }

        if (tb[TCA_ROUTE4_FROM] && tb[TCA_ROUTE4_IIF]) {
                NL_SET_ERR_MSG_ATTR(extack, tb[TCA_ROUTE4_FROM],
                                    "'from' and 'fromif' are mutually exclusive");
                return -EINVAL;
        }

        if (tb[TCA_ROUTE4_FROM]) {
                id = nla_get_u32(tb[TCA_ROUTE4_FROM]);
                nhandle |= id << 16;
        } else if (tb[TCA_ROUTE4_IIF]) {
                id = nla_get_u32(tb[TCA_ROUTE4_IIF]);
                nhandle |= (id | 0x8000) << 16;
        } else
                nhandle |= 0xFFFF << 16;

        if (handle && new) {
                nhandle |= handle & 0x7F00;
                if (nhandle != handle) {
                        NL_SET_ERR_MSG_FMT(extack,
                                           "Handle mismatch constructed: %x (expected: %x)",
                                           handle, nhandle);
                        return -EINVAL;
                }
        }

        if (!nhandle) {
                NL_SET_ERR_MSG(extack, "Replacing with handle of 0 is invalid");
                return -EINVAL;
        }

        h1 = to_hash(nhandle);
        b = rtnl_dereference(head->table[h1]);
        if (!b) {
                b = kzalloc(sizeof(struct route4_bucket), GFP_KERNEL);
                if (b == NULL)
                        return -ENOBUFS;

                rcu_assign_pointer(head->table[h1], b);
        } else {
                unsigned int h2 = from_hash(nhandle >> 16);

                for (fp = rtnl_dereference(b->ht[h2]);
                     fp;
                     fp = rtnl_dereference(fp->next))
                        if (fp->handle == f->handle)
                                return -EEXIST;
        }

        if (tb[TCA_ROUTE4_TO])
                f->id = to;

        if (tb[TCA_ROUTE4_FROM])
                f->id = to | id<<16;
        else if (tb[TCA_ROUTE4_IIF])
                f->iif = id;

        f->handle = nhandle;
        f->bkt = b;
        f->tp = tp;

        if (tb[TCA_ROUTE4_CLASSID]) {
                f->res.classid = nla_get_u32(tb[TCA_ROUTE4_CLASSID]);
                tcf_bind_filter(tp, &f->res, base);
        }

        return 0;
}

static int route4_change(struct net *net, struct sk_buff *in_skb,
                         struct tcf_proto *tp, unsigned long base, u32 handle,
                         struct nlattr **tca, void **arg, u32 flags,
                         struct netlink_ext_ack *extack)
{
        struct route4_head *head = rtnl_dereference(tp->root);
        struct route4_filter __rcu **fp;
        struct route4_filter *fold, *f1, *pfp, *f = NULL;
        struct route4_bucket *b;
        struct nlattr *tb[TCA_ROUTE4_MAX + 1];
        unsigned int h, th;
        int err;
        bool new = true;

        if (!handle) {
                NL_SET_ERR_MSG(extack, "Creating with handle of 0 is invalid");
                return -EINVAL;
        }

        if (NL_REQ_ATTR_CHECK(extack, NULL, tca, TCA_OPTIONS)) {
                NL_SET_ERR_MSG_MOD(extack, "Missing options");
                return -EINVAL;
        }

        err = nla_parse_nested_deprecated(tb, TCA_ROUTE4_MAX, tca[TCA_OPTIONS],
                                          route4_policy, NULL);
        if (err < 0)
                return err;

        fold = *arg;
        if (fold && fold->handle != handle)
                        return -EINVAL;

        err = -ENOBUFS;
        f = kzalloc(sizeof(struct route4_filter), GFP_KERNEL);
        if (!f)
                goto errout;

        err = tcf_exts_init(&f->exts, net, TCA_ROUTE4_ACT, TCA_ROUTE4_POLICE);
        if (err < 0)
                goto errout;

        if (fold) {
                f->id = fold->id;
                f->iif = fold->iif;
                f->handle = fold->handle;

                f->tp = fold->tp;
                f->bkt = fold->bkt;
                new = false;
        }

        err = route4_set_parms(net, tp, base, f, handle, head, tb,
                               tca[TCA_RATE], new, flags, extack);
        if (err < 0)
                goto errout;

        h = from_hash(f->handle >> 16);
        fp = &f->bkt->ht[h];
        for (pfp = rtnl_dereference(*fp);
             (f1 = rtnl_dereference(*fp)) != NULL;
             fp = &f1->next)
                if (f->handle < f1->handle)
                        break;

        tcf_block_netif_keep_dst(tp->chain->block);
        rcu_assign_pointer(f->next, f1);
        rcu_assign_pointer(*fp, f);

        if (fold) {
                th = to_hash(fold->handle);
                h = from_hash(fold->handle >> 16);
                b = rtnl_dereference(head->table[th]);
                if (b) {
                        fp = &b->ht[h];
                        for (pfp = rtnl_dereference(*fp); pfp;
                             fp = &pfp->next, pfp = rtnl_dereference(*fp)) {
                                if (pfp == fold) {
                                        rcu_assign_pointer(*fp, fold->next);
                                        break;
                                }
                        }
                }
        }

        route4_reset_fastmap(head);
        *arg = f;
        if (fold) {
                tcf_unbind_filter(tp, &fold->res);
                tcf_exts_get_net(&fold->exts);
                tcf_queue_work(&fold->rwork, route4_delete_filter_work);
        }
        return 0;

errout:
        if (f)
                tcf_exts_destroy(&f->exts);
        kfree(f);
        return err;
}

static void route4_walk(struct tcf_proto *tp, struct tcf_walker *arg,
                        bool rtnl_held)
{
        struct route4_head *head = rtnl_dereference(tp->root);
        unsigned int h, h1;

        if (head == NULL || arg->stop)
                return;

        for (h = 0; h <= 256; h++) {
                struct route4_bucket *b = rtnl_dereference(head->table[h]);

                if (b) {
                        for (h1 = 0; h1 <= 32; h1++) {
                                struct route4_filter *f;

                                for (f = rtnl_dereference(b->ht[h1]);
                                     f;
                                     f = rtnl_dereference(f->next)) {
                                        if (!tc_cls_stats_dump(tp, arg, f))
                                                return;
                                }
                        }
                }
        }
}

static int route4_dump(struct net *net, struct tcf_proto *tp, void *fh,
                       struct sk_buff *skb, struct tcmsg *t, bool rtnl_held)
{
        struct route4_filter *f = fh;
        struct nlattr *nest;
        u32 id;

        if (f == NULL)
                return skb->len;

        t->tcm_handle = f->handle;

        nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
        if (nest == NULL)
                goto nla_put_failure;

        if (!(f->handle & 0x8000)) {
                id = f->id & 0xFF;
                if (nla_put_u32(skb, TCA_ROUTE4_TO, id))
                        goto nla_put_failure;
        }
        if (f->handle & 0x80000000) {
                if ((f->handle >> 16) != 0xFFFF &&
                    nla_put_u32(skb, TCA_ROUTE4_IIF, f->iif))
                        goto nla_put_failure;
        } else {
                id = f->id >> 16;
                if (nla_put_u32(skb, TCA_ROUTE4_FROM, id))
                        goto nla_put_failure;
        }
        if (f->res.classid &&
            nla_put_u32(skb, TCA_ROUTE4_CLASSID, f->res.classid))
                goto nla_put_failure;

        if (tcf_exts_dump(skb, &f->exts) < 0)
                goto nla_put_failure;

        nla_nest_end(skb, nest);

        if (tcf_exts_dump_stats(skb, &f->exts) < 0)
                goto nla_put_failure;

        return skb->len;

nla_put_failure:
        nla_nest_cancel(skb, nest);
        return -1;
}

static void route4_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
                              unsigned long base)
{
        struct route4_filter *f = fh;

        tc_cls_bind_class(classid, cl, q, &f->res, base);
}

static struct tcf_proto_ops cls_route4_ops __read_mostly = {
        .kind                =        "route",
        .classify        =        route4_classify,
        .init                =        route4_init,
        .destroy        =        route4_destroy,
        .get                =        route4_get,
        .change                =        route4_change,
        .delete                =        route4_delete,
        .walk                =        route4_walk,
        .dump                =        route4_dump,
        .bind_class        =        route4_bind_class,
        .owner                =        THIS_MODULE,
};
MODULE_ALIAS_NET_CLS("route");

static int __init init_route4(void)
{
        return register_tcf_proto_ops(&cls_route4_ops);
}

static void __exit exit_route4(void)
{
        unregister_tcf_proto_ops(&cls_route4_ops);
}

module_init(init_route4)
module_exit(exit_route4)
MODULE_DESCRIPTION("Routing table realm based TC classifier");
MODULE_LICENSE("GPL");








































































    5 





















    3 














































  941 




















































  495 














































 17121 

























































 17028 





 1850 








1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
/* SPDX-License-Identifier: GPL-2.0 */
#undef TRACE_SYSTEM
#define TRACE_SYSTEM sock

#if !defined(_TRACE_SOCK_H) || defined(TRACE_HEADER_MULTI_READ)
#define _TRACE_SOCK_H

#include <net/sock.h>
#include <net/ipv6.h>
#include <linux/tracepoint.h>
#include <linux/ipv6.h>
#include <linux/tcp.h>
#include <trace/events/net_probe_common.h>

#define family_names                        \
                EM(AF_INET)                                \
                EMe(AF_INET6)

/* The protocol traced by inet_sock_set_state */
#define inet_protocol_names                \
                EM(IPPROTO_TCP)                        \
                EM(IPPROTO_DCCP)                \
                EM(IPPROTO_SCTP)                \
                EMe(IPPROTO_MPTCP)

#define tcp_state_names                        \
                EM(TCP_ESTABLISHED)                \
                EM(TCP_SYN_SENT)                \
                EM(TCP_SYN_RECV)                \
                EM(TCP_FIN_WAIT1)                \
                EM(TCP_FIN_WAIT2)                \
                EM(TCP_TIME_WAIT)                \
                EM(TCP_CLOSE)                        \
                EM(TCP_CLOSE_WAIT)                \
                EM(TCP_LAST_ACK)                \
                EM(TCP_LISTEN)                        \
                EM(TCP_CLOSING)                        \
                EMe(TCP_NEW_SYN_RECV)

#define skmem_kind_names                        \
                EM(SK_MEM_SEND)                        \
                EMe(SK_MEM_RECV)

/* enums need to be exported to user space */
#undef EM
#undef EMe
#define EM(a)       TRACE_DEFINE_ENUM(a);
#define EMe(a)      TRACE_DEFINE_ENUM(a);

family_names
inet_protocol_names
tcp_state_names
skmem_kind_names

#undef EM
#undef EMe
#define EM(a)       { a, #a },
#define EMe(a)      { a, #a }

#define show_family_name(val)                        \
        __print_symbolic(val, family_names)

#define show_inet_protocol_name(val)    \
        __print_symbolic(val, inet_protocol_names)

#define show_tcp_state_name(val)        \
        __print_symbolic(val, tcp_state_names)

#define show_skmem_kind_names(val)        \
        __print_symbolic(val, skmem_kind_names)

TRACE_EVENT(sock_rcvqueue_full,

        TP_PROTO(struct sock *sk, struct sk_buff *skb),

        TP_ARGS(sk, skb),

        TP_STRUCT__entry(
                __field(int, rmem_alloc)
                __field(unsigned int, truesize)
                __field(int, sk_rcvbuf)
        ),

        TP_fast_assign(
                __entry->rmem_alloc = atomic_read(&sk->sk_rmem_alloc);
                __entry->truesize   = skb->truesize;
                __entry->sk_rcvbuf  = READ_ONCE(sk->sk_rcvbuf);
        ),

        TP_printk("rmem_alloc=%d truesize=%u sk_rcvbuf=%d",
                __entry->rmem_alloc, __entry->truesize, __entry->sk_rcvbuf)
);

TRACE_EVENT(sock_exceed_buf_limit,

        TP_PROTO(struct sock *sk, struct proto *prot, long allocated, int kind),

        TP_ARGS(sk, prot, allocated, kind),

        TP_STRUCT__entry(
                __array(char, name, 32)
                __array(long, sysctl_mem, 3)
                __field(long, allocated)
                __field(int, sysctl_rmem)
                __field(int, rmem_alloc)
                __field(int, sysctl_wmem)
                __field(int, wmem_alloc)
                __field(int, wmem_queued)
                __field(int, kind)
        ),

        TP_fast_assign(
                strscpy(__entry->name, prot->name, 32);
                __entry->sysctl_mem[0] = READ_ONCE(prot->sysctl_mem[0]);
                __entry->sysctl_mem[1] = READ_ONCE(prot->sysctl_mem[1]);
                __entry->sysctl_mem[2] = READ_ONCE(prot->sysctl_mem[2]);
                __entry->allocated = allocated;
                __entry->sysctl_rmem = sk_get_rmem0(sk, prot);
                __entry->rmem_alloc = atomic_read(&sk->sk_rmem_alloc);
                __entry->sysctl_wmem = sk_get_wmem0(sk, prot);
                __entry->wmem_alloc = refcount_read(&sk->sk_wmem_alloc);
                __entry->wmem_queued = READ_ONCE(sk->sk_wmem_queued);
                __entry->kind = kind;
        ),

        TP_printk("proto:%s sysctl_mem=%ld,%ld,%ld allocated=%ld sysctl_rmem=%d rmem_alloc=%d sysctl_wmem=%d wmem_alloc=%d wmem_queued=%d kind=%s",
                __entry->name,
                __entry->sysctl_mem[0],
                __entry->sysctl_mem[1],
                __entry->sysctl_mem[2],
                __entry->allocated,
                __entry->sysctl_rmem,
                __entry->rmem_alloc,
                __entry->sysctl_wmem,
                __entry->wmem_alloc,
                __entry->wmem_queued,
                show_skmem_kind_names(__entry->kind)
        )
);

TRACE_EVENT(inet_sock_set_state,

        TP_PROTO(const struct sock *sk, const int oldstate, const int newstate),

        TP_ARGS(sk, oldstate, newstate),

        TP_STRUCT__entry(
                __field(const void *, skaddr)
                __field(int, oldstate)
                __field(int, newstate)
                __field(__u16, sport)
                __field(__u16, dport)
                __field(__u16, family)
                __field(__u16, protocol)
                __array(__u8, saddr, 4)
                __array(__u8, daddr, 4)
                __array(__u8, saddr_v6, 16)
                __array(__u8, daddr_v6, 16)
        ),

        TP_fast_assign(
                const struct inet_sock *inet = inet_sk(sk);
                __be32 *p32;

                __entry->skaddr = sk;
                __entry->oldstate = oldstate;
                __entry->newstate = newstate;

                __entry->family = sk->sk_family;
                __entry->protocol = sk->sk_protocol;
                __entry->sport = ntohs(inet->inet_sport);
                __entry->dport = ntohs(inet->inet_dport);

                p32 = (__be32 *) __entry->saddr;
                *p32 = inet->inet_saddr;

                p32 = (__be32 *) __entry->daddr;
                *p32 =  inet->inet_daddr;

                TP_STORE_ADDRS(__entry, inet->inet_saddr, inet->inet_daddr,
                               sk->sk_v6_rcv_saddr, sk->sk_v6_daddr);
        ),

        TP_printk("family=%s protocol=%s sport=%hu dport=%hu saddr=%pI4 daddr=%pI4 saddrv6=%pI6c daddrv6=%pI6c oldstate=%s newstate=%s",
                        show_family_name(__entry->family),
                        show_inet_protocol_name(__entry->protocol),
                        __entry->sport, __entry->dport,
                        __entry->saddr, __entry->daddr,
                        __entry->saddr_v6, __entry->daddr_v6,
                        show_tcp_state_name(__entry->oldstate),
                        show_tcp_state_name(__entry->newstate))
);

TRACE_EVENT(inet_sk_error_report,

        TP_PROTO(const struct sock *sk),

        TP_ARGS(sk),

        TP_STRUCT__entry(
                __field(int, error)
                __field(__u16, sport)
                __field(__u16, dport)
                __field(__u16, family)
                __field(__u16, protocol)
                __array(__u8, saddr, 4)
                __array(__u8, daddr, 4)
                __array(__u8, saddr_v6, 16)
                __array(__u8, daddr_v6, 16)
        ),

        TP_fast_assign(
                const struct inet_sock *inet = inet_sk(sk);
                __be32 *p32;

                __entry->error = sk->sk_err;
                __entry->family = sk->sk_family;
                __entry->protocol = sk->sk_protocol;
                __entry->sport = ntohs(inet->inet_sport);
                __entry->dport = ntohs(inet->inet_dport);

                p32 = (__be32 *) __entry->saddr;
                *p32 = inet->inet_saddr;

                p32 = (__be32 *) __entry->daddr;
                *p32 =  inet->inet_daddr;

                TP_STORE_ADDRS(__entry, inet->inet_saddr, inet->inet_daddr,
                               sk->sk_v6_rcv_saddr, sk->sk_v6_daddr);
        ),

        TP_printk("family=%s protocol=%s sport=%hu dport=%hu saddr=%pI4 daddr=%pI4 saddrv6=%pI6c daddrv6=%pI6c error=%d",
                  show_family_name(__entry->family),
                  show_inet_protocol_name(__entry->protocol),
                  __entry->sport, __entry->dport,
                  __entry->saddr, __entry->daddr,
                  __entry->saddr_v6, __entry->daddr_v6,
                  __entry->error)
);

TRACE_EVENT(sk_data_ready,

        TP_PROTO(const struct sock *sk),

        TP_ARGS(sk),

        TP_STRUCT__entry(
                __field(const void *, skaddr)
                __field(__u16, family)
                __field(__u16, protocol)
                __field(unsigned long, ip)
        ),

        TP_fast_assign(
                __entry->skaddr = sk;
                __entry->family = sk->sk_family;
                __entry->protocol = sk->sk_protocol;
                __entry->ip = _RET_IP_;
        ),

        TP_printk("family=%u protocol=%u func=%ps",
                  __entry->family, __entry->protocol, (void *)__entry->ip)
);

/*
 * sock send/recv msg length
 */
DECLARE_EVENT_CLASS(sock_msg_length,

        TP_PROTO(struct sock *sk, int ret, int flags),

        TP_ARGS(sk, ret, flags),

        TP_STRUCT__entry(
                __field(void *, sk)
                __field(__u16, family)
                __field(__u16, protocol)
                __field(int, ret)
                __field(int, flags)
        ),

        TP_fast_assign(
                __entry->sk = sk;
                __entry->family = sk->sk_family;
                __entry->protocol = sk->sk_protocol;
                __entry->ret = ret;
                __entry->flags = flags;
        ),

        TP_printk("sk address = %p, family = %s protocol = %s, length = %d, error = %d, flags = 0x%x",
                  __entry->sk, show_family_name(__entry->family),
                  show_inet_protocol_name(__entry->protocol),
                  !(__entry->flags & MSG_PEEK) ?
                  (__entry->ret > 0 ? __entry->ret : 0) : 0,
                  __entry->ret < 0 ? __entry->ret : 0,
                  __entry->flags)
);

DEFINE_EVENT(sock_msg_length, sock_send_length,
        TP_PROTO(struct sock *sk, int ret, int flags),

        TP_ARGS(sk, ret, flags)
);

DEFINE_EVENT(sock_msg_length, sock_recv_length,
        TP_PROTO(struct sock *sk, int ret, int flags),

        TP_ARGS(sk, ret, flags)
);
#endif /* _TRACE_SOCK_H */

/* This part must be outside protection */
#include <trace/define_trace.h>





































    2 


















1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
/* file-mmu.c: ramfs MMU-based file operations
 *
 * Resizable simple ram filesystem for Linux.
 *
 * Copyright (C) 2000 Linus Torvalds.
 *               2000 Transmeta Corp.
 *
 * Usage limits added by David Gibson, Linuxcare Australia.
 * This file is released under the GPL.
 */

/*
 * NOTE! This filesystem is probably most useful
 * not as a real filesystem, but as an example of
 * how virtual filesystems can be written.
 *
 * It doesn't get much simpler than this. Consider
 * that this file implements the full semantics of
 * a POSIX-compliant read-write filesystem.
 *
 * Note in particular how the filesystem does not
 * need to implement any data structures of its own
 * to keep track of the virtual data: using the VFS
 * caches is sufficient.
 */

#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/ramfs.h>
#include <linux/sched.h>

#include "internal.h"

static unsigned long ramfs_mmu_get_unmapped_area(struct file *file,
                unsigned long addr, unsigned long len, unsigned long pgoff,
                unsigned long flags)
{
        return mm_get_unmapped_area(current->mm, file, addr, len, pgoff, flags);
}

const struct file_operations ramfs_file_operations = {
        .read_iter        = generic_file_read_iter,
        .write_iter        = generic_file_write_iter,
        .mmap                = generic_file_mmap,
        .fsync                = noop_fsync,
        .splice_read        = filemap_splice_read,
        .splice_write        = iter_file_splice_write,
        .llseek                = generic_file_llseek,
        .get_unmapped_area        = ramfs_mmu_get_unmapped_area,
};

const struct inode_operations ramfs_file_inode_operations = {
        .setattr        = simple_setattr,
        .getattr        = simple_getattr,
};



















































































































  284 









































  212 

   36 
























    1 













1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the BSD Socket
 *                interface as the means of communication with the user level.
 *
 * Authors:        Lotsa people, from code originally in tcp
 */

#ifndef _INET6_HASHTABLES_H
#define _INET6_HASHTABLES_H


#if IS_ENABLED(CONFIG_IPV6)
#include <linux/in6.h>
#include <linux/ipv6.h>
#include <linux/types.h>
#include <linux/jhash.h>

#include <net/inet_sock.h>

#include <net/ipv6.h>
#include <net/netns/hash.h>

struct inet_hashinfo;

static inline unsigned int __inet6_ehashfn(const u32 lhash,
                                    const u16 lport,
                                    const u32 fhash,
                                    const __be16 fport,
                                    const u32 initval)
{
        const u32 ports = (((u32)lport) << 16) | (__force u32)fport;
        return jhash_3words(lhash, fhash, ports, initval);
}

/*
 * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
 * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM
 *
 * The sockhash lock must be held as a reader here.
 */
struct sock *__inet6_lookup_established(const struct net *net,
                                        struct inet_hashinfo *hashinfo,
                                        const struct in6_addr *saddr,
                                        const __be16 sport,
                                        const struct in6_addr *daddr,
                                        const u16 hnum, const int dif,
                                        const int sdif);

typedef u32 (inet6_ehashfn_t)(const struct net *net,
                               const struct in6_addr *laddr, const u16 lport,
                               const struct in6_addr *faddr, const __be16 fport);

inet6_ehashfn_t inet6_ehashfn;

INDIRECT_CALLABLE_DECLARE(inet6_ehashfn_t udp6_ehashfn);

struct sock *inet6_lookup_reuseport(const struct net *net, struct sock *sk,
                                    struct sk_buff *skb, int doff,
                                    const struct in6_addr *saddr,
                                    __be16 sport,
                                    const struct in6_addr *daddr,
                                    unsigned short hnum,
                                    inet6_ehashfn_t *ehashfn);

struct sock *inet6_lookup_listener(const struct net *net,
                                   struct inet_hashinfo *hashinfo,
                                   struct sk_buff *skb, int doff,
                                   const struct in6_addr *saddr,
                                   const __be16 sport,
                                   const struct in6_addr *daddr,
                                   const unsigned short hnum,
                                   const int dif, const int sdif);

struct sock *inet6_lookup_run_sk_lookup(const struct net *net,
                                        int protocol,
                                        struct sk_buff *skb, int doff,
                                        const struct in6_addr *saddr,
                                        const __be16 sport,
                                        const struct in6_addr *daddr,
                                        const u16 hnum, const int dif,
                                        inet6_ehashfn_t *ehashfn);

static inline struct sock *__inet6_lookup(const struct net *net,
                                          struct inet_hashinfo *hashinfo,
                                          struct sk_buff *skb, int doff,
                                          const struct in6_addr *saddr,
                                          const __be16 sport,
                                          const struct in6_addr *daddr,
                                          const u16 hnum,
                                          const int dif, const int sdif,
                                          bool *refcounted)
{
        struct sock *sk = __inet6_lookup_established(net, hashinfo, saddr,
                                                     sport, daddr, hnum,
                                                     dif, sdif);
        *refcounted = true;
        if (sk)
                return sk;
        *refcounted = false;
        return inet6_lookup_listener(net, hashinfo, skb, doff, saddr, sport,
                                     daddr, hnum, dif, sdif);
}

static inline
struct sock *inet6_steal_sock(struct net *net, struct sk_buff *skb, int doff,
                              const struct in6_addr *saddr, const __be16 sport,
                              const struct in6_addr *daddr, const __be16 dport,
                              bool *refcounted, inet6_ehashfn_t *ehashfn)
{
        struct sock *sk, *reuse_sk;
        bool prefetched;

        sk = skb_steal_sock(skb, refcounted, &prefetched);
        if (!sk)
                return NULL;

        if (!prefetched || !sk_fullsock(sk))
                return sk;

        if (sk->sk_protocol == IPPROTO_TCP) {
                if (sk->sk_state != TCP_LISTEN)
                        return sk;
        } else if (sk->sk_protocol == IPPROTO_UDP) {
                if (sk->sk_state != TCP_CLOSE)
                        return sk;
        } else {
                return sk;
        }

        reuse_sk = inet6_lookup_reuseport(net, sk, skb, doff,
                                          saddr, sport, daddr, ntohs(dport),
                                          ehashfn);
        if (!reuse_sk)
                return sk;

        /* We've chosen a new reuseport sock which is never refcounted. This
         * implies that sk also isn't refcounted.
         */
        WARN_ON_ONCE(*refcounted);

        return reuse_sk;
}

static inline struct sock *__inet6_lookup_skb(struct inet_hashinfo *hashinfo,
                                              struct sk_buff *skb, int doff,
                                              const __be16 sport,
                                              const __be16 dport,
                                              int iif, int sdif,
                                              bool *refcounted)
{
        struct net *net = dev_net(skb_dst(skb)->dev);
        const struct ipv6hdr *ip6h = ipv6_hdr(skb);
        struct sock *sk;

        sk = inet6_steal_sock(net, skb, doff, &ip6h->saddr, sport, &ip6h->daddr, dport,
                              refcounted, inet6_ehashfn);
        if (IS_ERR(sk))
                return NULL;
        if (sk)
                return sk;

        return __inet6_lookup(net, hashinfo, skb,
                              doff, &ip6h->saddr, sport,
                              &ip6h->daddr, ntohs(dport),
                              iif, sdif, refcounted);
}

struct sock *inet6_lookup(const struct net *net, struct inet_hashinfo *hashinfo,
                          struct sk_buff *skb, int doff,
                          const struct in6_addr *saddr, const __be16 sport,
                          const struct in6_addr *daddr, const __be16 dport,
                          const int dif);

int inet6_hash(struct sock *sk);

static inline bool inet6_match(const struct net *net, const struct sock *sk,
                               const struct in6_addr *saddr,
                               const struct in6_addr *daddr,
                               const __portpair ports,
                               const int dif, const int sdif)
{
        if (!net_eq(sock_net(sk), net) ||
            sk->sk_family != AF_INET6 ||
            sk->sk_portpair != ports ||
            !ipv6_addr_equal(&sk->sk_v6_daddr, saddr) ||
            !ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))
                return false;

        /* READ_ONCE() paired with WRITE_ONCE() in sock_bindtoindex_locked() */
        return inet_sk_bound_dev_eq(net, READ_ONCE(sk->sk_bound_dev_if), dif,
                                    sdif);
}
#endif /* IS_ENABLED(CONFIG_IPV6) */

#endif /* _INET6_HASHTABLES_H */























































































































    1 
   25 





   20 

    8 








   20 
   11 







    4 
   13 


    9 

   16 




   20 
    9 






   19 


   20 



   20 




   18 








   37 







    2 






    2 





    5 
    3 

    2 






    4 








    4 


















































   26 








   20 

   26 


   22 


   26 








   15 


   20 








   26 



   20 

   10 




   12 















   12 

   12 








   12 




   12 


























   12 













   12 






   19 


   19 













   15 








   12 




































   15 

    5 


   15 











   18 






   18 

    4 


   15 





























    4 













































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
// SPDX-License-Identifier: GPL-2.0

#include "misc.h"
#include "ctree.h"
#include "block-rsv.h"
#include "space-info.h"
#include "transaction.h"
#include "block-group.h"
#include "fs.h"
#include "accessors.h"

/*
 * HOW DO BLOCK RESERVES WORK
 *
 *   Think of block_rsv's as buckets for logically grouped metadata
 *   reservations.  Each block_rsv has a ->size and a ->reserved.  ->size is
 *   how large we want our block rsv to be, ->reserved is how much space is
 *   currently reserved for this block reserve.
 *
 *   ->failfast exists for the truncate case, and is described below.
 *
 * NORMAL OPERATION
 *
 *   -> Reserve
 *     Entrance: btrfs_block_rsv_add, btrfs_block_rsv_refill
 *
 *     We call into btrfs_reserve_metadata_bytes() with our bytes, which is
 *     accounted for in space_info->bytes_may_use, and then add the bytes to
 *     ->reserved, and ->size in the case of btrfs_block_rsv_add.
 *
 *     ->size is an over-estimation of how much we may use for a particular
 *     operation.
 *
 *   -> Use
 *     Entrance: btrfs_use_block_rsv
 *
 *     When we do a btrfs_alloc_tree_block() we call into btrfs_use_block_rsv()
 *     to determine the appropriate block_rsv to use, and then verify that
 *     ->reserved has enough space for our tree block allocation.  Once
 *     successful we subtract fs_info->nodesize from ->reserved.
 *
 *   -> Finish
 *     Entrance: btrfs_block_rsv_release
 *
 *     We are finished with our operation, subtract our individual reservation
 *     from ->size, and then subtract ->size from ->reserved and free up the
 *     excess if there is any.
 *
 *     There is some logic here to refill the delayed refs rsv or the global rsv
 *     as needed, otherwise the excess is subtracted from
 *     space_info->bytes_may_use.
 *
 * TYPES OF BLOCK RESERVES
 *
 * BLOCK_RSV_TRANS, BLOCK_RSV_DELOPS, BLOCK_RSV_CHUNK
 *   These behave normally, as described above, just within the confines of the
 *   lifetime of their particular operation (transaction for the whole trans
 *   handle lifetime, for example).
 *
 * BLOCK_RSV_GLOBAL
 *   It is impossible to properly account for all the space that may be required
 *   to make our extent tree updates.  This block reserve acts as an overflow
 *   buffer in case our delayed refs reserve does not reserve enough space to
 *   update the extent tree.
 *
 *   We can steal from this in some cases as well, notably on evict() or
 *   truncate() in order to help users recover from ENOSPC conditions.
 *
 * BLOCK_RSV_DELALLOC
 *   The individual item sizes are determined by the per-inode size
 *   calculations, which are described with the delalloc code.  This is pretty
 *   straightforward, it's just the calculation of ->size encodes a lot of
 *   different items, and thus it gets used when updating inodes, inserting file
 *   extents, and inserting checksums.
 *
 * BLOCK_RSV_DELREFS
 *   We keep a running tally of how many delayed refs we have on the system.
 *   We assume each one of these delayed refs are going to use a full
 *   reservation.  We use the transaction items and pre-reserve space for every
 *   operation, and use this reservation to refill any gap between ->size and
 *   ->reserved that may exist.
 *
 *   From there it's straightforward, removing a delayed ref means we remove its
 *   count from ->size and free up reservations as necessary.  Since this is
 *   the most dynamic block reserve in the system, we will try to refill this
 *   block reserve first with any excess returned by any other block reserve.
 *
 * BLOCK_RSV_EMPTY
 *   This is the fallback block reserve to make us try to reserve space if we
 *   don't have a specific bucket for this allocation.  It is mostly used for
 *   updating the device tree and such, since that is a separate pool we're
 *   content to just reserve space from the space_info on demand.
 *
 * BLOCK_RSV_TEMP
 *   This is used by things like truncate and iput.  We will temporarily
 *   allocate a block reserve, set it to some size, and then truncate bytes
 *   until we have no space left.  With ->failfast set we'll simply return
 *   ENOSPC from btrfs_use_block_rsv() to signal that we need to unwind and try
 *   to make a new reservation.  This is because these operations are
 *   unbounded, so we want to do as much work as we can, and then back off and
 *   re-reserve.
 */

static u64 block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
                                    struct btrfs_block_rsv *block_rsv,
                                    struct btrfs_block_rsv *dest, u64 num_bytes,
                                    u64 *qgroup_to_release_ret)
{
        struct btrfs_space_info *space_info = block_rsv->space_info;
        u64 qgroup_to_release = 0;
        u64 ret;

        spin_lock(&block_rsv->lock);
        if (num_bytes == (u64)-1) {
                num_bytes = block_rsv->size;
                qgroup_to_release = block_rsv->qgroup_rsv_size;
        }
        block_rsv->size -= num_bytes;
        if (block_rsv->reserved >= block_rsv->size) {
                num_bytes = block_rsv->reserved - block_rsv->size;
                block_rsv->reserved = block_rsv->size;
                block_rsv->full = true;
        } else {
                num_bytes = 0;
        }
        if (qgroup_to_release_ret &&
            block_rsv->qgroup_rsv_reserved >= block_rsv->qgroup_rsv_size) {
                qgroup_to_release = block_rsv->qgroup_rsv_reserved -
                                    block_rsv->qgroup_rsv_size;
                block_rsv->qgroup_rsv_reserved = block_rsv->qgroup_rsv_size;
        } else {
                qgroup_to_release = 0;
        }
        spin_unlock(&block_rsv->lock);

        ret = num_bytes;
        if (num_bytes > 0) {
                if (dest) {
                        spin_lock(&dest->lock);
                        if (!dest->full) {
                                u64 bytes_to_add;

                                bytes_to_add = dest->size - dest->reserved;
                                bytes_to_add = min(num_bytes, bytes_to_add);
                                dest->reserved += bytes_to_add;
                                if (dest->reserved >= dest->size)
                                        dest->full = true;
                                num_bytes -= bytes_to_add;
                        }
                        spin_unlock(&dest->lock);
                }
                if (num_bytes)
                        btrfs_space_info_free_bytes_may_use(fs_info,
                                                            space_info,
                                                            num_bytes);
        }
        if (qgroup_to_release_ret)
                *qgroup_to_release_ret = qgroup_to_release;
        return ret;
}

int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src,
                            struct btrfs_block_rsv *dst, u64 num_bytes,
                            bool update_size)
{
        int ret;

        ret = btrfs_block_rsv_use_bytes(src, num_bytes);
        if (ret)
                return ret;

        btrfs_block_rsv_add_bytes(dst, num_bytes, update_size);
        return 0;
}

void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, enum btrfs_rsv_type type)
{
        memset(rsv, 0, sizeof(*rsv));
        spin_lock_init(&rsv->lock);
        rsv->type = type;
}

void btrfs_init_metadata_block_rsv(struct btrfs_fs_info *fs_info,
                                   struct btrfs_block_rsv *rsv,
                                   enum btrfs_rsv_type type)
{
        btrfs_init_block_rsv(rsv, type);
        rsv->space_info = btrfs_find_space_info(fs_info,
                                            BTRFS_BLOCK_GROUP_METADATA);
}

struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_fs_info *fs_info,
                                              enum btrfs_rsv_type type)
{
        struct btrfs_block_rsv *block_rsv;

        block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
        if (!block_rsv)
                return NULL;

        btrfs_init_metadata_block_rsv(fs_info, block_rsv, type);
        return block_rsv;
}

void btrfs_free_block_rsv(struct btrfs_fs_info *fs_info,
                          struct btrfs_block_rsv *rsv)
{
        if (!rsv)
                return;
        btrfs_block_rsv_release(fs_info, rsv, (u64)-1, NULL);
        kfree(rsv);
}

int btrfs_block_rsv_add(struct btrfs_fs_info *fs_info,
                        struct btrfs_block_rsv *block_rsv, u64 num_bytes,
                        enum btrfs_reserve_flush_enum flush)
{
        int ret;

        if (num_bytes == 0)
                return 0;

        ret = btrfs_reserve_metadata_bytes(fs_info, block_rsv->space_info,
                                           num_bytes, flush);
        if (!ret)
                btrfs_block_rsv_add_bytes(block_rsv, num_bytes, true);

        return ret;
}

int btrfs_block_rsv_check(struct btrfs_block_rsv *block_rsv, int min_percent)
{
        u64 num_bytes = 0;
        int ret = -ENOSPC;

        spin_lock(&block_rsv->lock);
        num_bytes = mult_perc(block_rsv->size, min_percent);
        if (block_rsv->reserved >= num_bytes)
                ret = 0;
        spin_unlock(&block_rsv->lock);

        return ret;
}

int btrfs_block_rsv_refill(struct btrfs_fs_info *fs_info,
                           struct btrfs_block_rsv *block_rsv, u64 num_bytes,
                           enum btrfs_reserve_flush_enum flush)
{
        int ret = -ENOSPC;

        if (!block_rsv)
                return 0;

        spin_lock(&block_rsv->lock);
        if (block_rsv->reserved >= num_bytes)
                ret = 0;
        else
                num_bytes -= block_rsv->reserved;
        spin_unlock(&block_rsv->lock);

        if (!ret)
                return 0;

        ret = btrfs_reserve_metadata_bytes(fs_info, block_rsv->space_info,
                                           num_bytes, flush);
        if (!ret) {
                btrfs_block_rsv_add_bytes(block_rsv, num_bytes, false);
                return 0;
        }

        return ret;
}

u64 btrfs_block_rsv_release(struct btrfs_fs_info *fs_info,
                            struct btrfs_block_rsv *block_rsv, u64 num_bytes,
                            u64 *qgroup_to_release)
{
        struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
        struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_refs_rsv;
        struct btrfs_block_rsv *target = NULL;

        /*
         * If we are a delayed block reserve then push to the global rsv,
         * otherwise dump into the global delayed reserve if it is not full.
         */
        if (block_rsv->type == BTRFS_BLOCK_RSV_DELOPS)
                target = global_rsv;
        else if (block_rsv != global_rsv && !btrfs_block_rsv_full(delayed_rsv))
                target = delayed_rsv;

        if (target && block_rsv->space_info != target->space_info)
                target = NULL;

        return block_rsv_release_bytes(fs_info, block_rsv, target, num_bytes,
                                       qgroup_to_release);
}

int btrfs_block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv, u64 num_bytes)
{
        int ret = -ENOSPC;

        spin_lock(&block_rsv->lock);
        if (block_rsv->reserved >= num_bytes) {
                block_rsv->reserved -= num_bytes;
                if (block_rsv->reserved < block_rsv->size)
                        block_rsv->full = false;
                ret = 0;
        }
        spin_unlock(&block_rsv->lock);
        return ret;
}

void btrfs_block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
                               u64 num_bytes, bool update_size)
{
        spin_lock(&block_rsv->lock);
        block_rsv->reserved += num_bytes;
        if (update_size)
                block_rsv->size += num_bytes;
        else if (block_rsv->reserved >= block_rsv->size)
                block_rsv->full = true;
        spin_unlock(&block_rsv->lock);
}

void btrfs_update_global_block_rsv(struct btrfs_fs_info *fs_info)
{
        struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
        struct btrfs_space_info *sinfo = block_rsv->space_info;
        struct btrfs_root *root, *tmp;
        u64 num_bytes = btrfs_root_used(&fs_info->tree_root->root_item);
        unsigned int min_items = 1;

        /*
         * The global block rsv is based on the size of the extent tree, the
         * checksum tree and the root tree.  If the fs is empty we want to set
         * it to a minimal amount for safety.
         *
         * We also are going to need to modify the minimum of the tree root and
         * any global roots we could touch.
         */
        read_lock(&fs_info->global_root_lock);
        rbtree_postorder_for_each_entry_safe(root, tmp, &fs_info->global_root_tree,
                                             rb_node) {
                if (btrfs_root_id(root) == BTRFS_EXTENT_TREE_OBJECTID ||
                    btrfs_root_id(root) == BTRFS_CSUM_TREE_OBJECTID ||
                    btrfs_root_id(root) == BTRFS_FREE_SPACE_TREE_OBJECTID) {
                        num_bytes += btrfs_root_used(&root->root_item);
                        min_items++;
                }
        }
        read_unlock(&fs_info->global_root_lock);

        if (btrfs_fs_compat_ro(fs_info, BLOCK_GROUP_TREE)) {
                num_bytes += btrfs_root_used(&fs_info->block_group_root->root_item);
                min_items++;
        }

        if (btrfs_fs_incompat(fs_info, RAID_STRIPE_TREE)) {
                num_bytes += btrfs_root_used(&fs_info->stripe_root->root_item);
                min_items++;
        }

        /*
         * But we also want to reserve enough space so we can do the fallback
         * global reserve for an unlink, which is an additional
         * BTRFS_UNLINK_METADATA_UNITS items.
         *
         * But we also need space for the delayed ref updates from the unlink,
         * so add BTRFS_UNLINK_METADATA_UNITS units for delayed refs, one for
         * each unlink metadata item.
         */
        min_items += BTRFS_UNLINK_METADATA_UNITS;

        num_bytes = max_t(u64, num_bytes,
                          btrfs_calc_insert_metadata_size(fs_info, min_items) +
                          btrfs_calc_delayed_ref_bytes(fs_info,
                                               BTRFS_UNLINK_METADATA_UNITS));

        spin_lock(&sinfo->lock);
        spin_lock(&block_rsv->lock);

        block_rsv->size = min_t(u64, num_bytes, SZ_512M);

        if (block_rsv->reserved < block_rsv->size) {
                num_bytes = block_rsv->size - block_rsv->reserved;
                btrfs_space_info_update_bytes_may_use(fs_info, sinfo,
                                                      num_bytes);
                block_rsv->reserved = block_rsv->size;
        } else if (block_rsv->reserved > block_rsv->size) {
                num_bytes = block_rsv->reserved - block_rsv->size;
                btrfs_space_info_update_bytes_may_use(fs_info, sinfo,
                                                      -num_bytes);
                block_rsv->reserved = block_rsv->size;
                btrfs_try_granting_tickets(fs_info, sinfo);
        }

        block_rsv->full = (block_rsv->reserved == block_rsv->size);

        if (block_rsv->size >= sinfo->total_bytes)
                sinfo->force_alloc = CHUNK_ALLOC_FORCE;
        spin_unlock(&block_rsv->lock);
        spin_unlock(&sinfo->lock);
}

void btrfs_init_root_block_rsv(struct btrfs_root *root)
{
        struct btrfs_fs_info *fs_info = root->fs_info;

        switch (btrfs_root_id(root)) {
        case BTRFS_CSUM_TREE_OBJECTID:
        case BTRFS_EXTENT_TREE_OBJECTID:
        case BTRFS_FREE_SPACE_TREE_OBJECTID:
        case BTRFS_BLOCK_GROUP_TREE_OBJECTID:
        case BTRFS_RAID_STRIPE_TREE_OBJECTID:
                root->block_rsv = &fs_info->delayed_refs_rsv;
                break;
        case BTRFS_ROOT_TREE_OBJECTID:
        case BTRFS_DEV_TREE_OBJECTID:
        case BTRFS_QUOTA_TREE_OBJECTID:
                root->block_rsv = &fs_info->global_block_rsv;
                break;
        case BTRFS_CHUNK_TREE_OBJECTID:
                root->block_rsv = &fs_info->chunk_block_rsv;
                break;
        default:
                root->block_rsv = NULL;
                break;
        }
}

void btrfs_init_global_block_rsv(struct btrfs_fs_info *fs_info)
{
        struct btrfs_space_info *space_info;

        space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
        fs_info->chunk_block_rsv.space_info = space_info;

        space_info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
        fs_info->global_block_rsv.space_info = space_info;
        fs_info->trans_block_rsv.space_info = space_info;
        fs_info->empty_block_rsv.space_info = space_info;
        fs_info->delayed_block_rsv.space_info = space_info;
        fs_info->delayed_refs_rsv.space_info = space_info;

        btrfs_update_global_block_rsv(fs_info);
}

void btrfs_release_global_block_rsv(struct btrfs_fs_info *fs_info)
{
        btrfs_block_rsv_release(fs_info, &fs_info->global_block_rsv, (u64)-1,
                                NULL);
        WARN_ON(fs_info->trans_block_rsv.size > 0);
        WARN_ON(fs_info->trans_block_rsv.reserved > 0);
        WARN_ON(fs_info->chunk_block_rsv.size > 0);
        WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
        WARN_ON(fs_info->delayed_block_rsv.size > 0);
        WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
        WARN_ON(fs_info->delayed_refs_rsv.reserved > 0);
        WARN_ON(fs_info->delayed_refs_rsv.size > 0);
}

static struct btrfs_block_rsv *get_block_rsv(
                                        const struct btrfs_trans_handle *trans,
                                        const struct btrfs_root *root)
{
        struct btrfs_fs_info *fs_info = root->fs_info;
        struct btrfs_block_rsv *block_rsv = NULL;

        if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state) ||
            (root == fs_info->uuid_root) ||
            (trans->adding_csums && btrfs_root_id(root) == BTRFS_CSUM_TREE_OBJECTID))
                block_rsv = trans->block_rsv;

        if (!block_rsv)
                block_rsv = root->block_rsv;

        if (!block_rsv)
                block_rsv = &fs_info->empty_block_rsv;

        return block_rsv;
}

struct btrfs_block_rsv *btrfs_use_block_rsv(struct btrfs_trans_handle *trans,
                                            struct btrfs_root *root,
                                            u32 blocksize)
{
        struct btrfs_fs_info *fs_info = root->fs_info;
        struct btrfs_block_rsv *block_rsv;
        struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
        int ret;
        bool global_updated = false;

        block_rsv = get_block_rsv(trans, root);

        if (unlikely(btrfs_block_rsv_size(block_rsv) == 0))
                goto try_reserve;
again:
        ret = btrfs_block_rsv_use_bytes(block_rsv, blocksize);
        if (!ret)
                return block_rsv;

        if (block_rsv->failfast)
                return ERR_PTR(ret);

        if (block_rsv->type == BTRFS_BLOCK_RSV_GLOBAL && !global_updated) {
                global_updated = true;
                btrfs_update_global_block_rsv(fs_info);
                goto again;
        }

        /*
         * The global reserve still exists to save us from ourselves, so don't
         * warn_on if we are short on our delayed refs reserve.
         */
        if (block_rsv->type != BTRFS_BLOCK_RSV_DELREFS &&
            btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
                static DEFINE_RATELIMIT_STATE(_rs,
                                DEFAULT_RATELIMIT_INTERVAL * 10,
                                /*DEFAULT_RATELIMIT_BURST*/ 1);
                if (__ratelimit(&_rs))
                        WARN(1, KERN_DEBUG
                                "BTRFS: block rsv %d returned %d\n",
                                block_rsv->type, ret);
        }
try_reserve:
        ret = btrfs_reserve_metadata_bytes(fs_info, block_rsv->space_info,
                                           blocksize, BTRFS_RESERVE_NO_FLUSH);
        if (!ret)
                return block_rsv;
        /*
         * If we couldn't reserve metadata bytes try and use some from
         * the global reserve if its space type is the same as the global
         * reservation.
         */
        if (block_rsv->type != BTRFS_BLOCK_RSV_GLOBAL &&
            block_rsv->space_info == global_rsv->space_info) {
                ret = btrfs_block_rsv_use_bytes(global_rsv, blocksize);
                if (!ret)
                        return global_rsv;
        }

        /*
         * All hope is lost, but of course our reservations are overly
         * pessimistic, so instead of possibly having an ENOSPC abort here, try
         * one last time to force a reservation if there's enough actual space
         * on disk to make the reservation.
         */
        ret = btrfs_reserve_metadata_bytes(fs_info, block_rsv->space_info, blocksize,
                                           BTRFS_RESERVE_FLUSH_EMERGENCY);
        if (!ret)
                return block_rsv;

        return ERR_PTR(ret);
}

int btrfs_check_trunc_cache_free_space(const struct btrfs_fs_info *fs_info,
                                       struct btrfs_block_rsv *rsv)
{
        u64 needed_bytes;
        int ret;

        /* 1 for slack space, 1 for updating the inode */
        needed_bytes = btrfs_calc_insert_metadata_size(fs_info, 1) +
                btrfs_calc_metadata_size(fs_info, 1);

        spin_lock(&rsv->lock);
        if (rsv->reserved < needed_bytes)
                ret = -ENOSPC;
        else
                ret = 0;
        spin_unlock(&rsv->lock);
        return ret;
}





































































































































































































    1 

















    1 


























































































    1 

















































































































































































































































































































































































































    1 





























































































































































































    2 









    1 
    1 























































    1 



















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
// SPDX-License-Identifier: GPL-2.0-only
/*
 * CAN driver for "8 devices" USB2CAN converter
 *
 * Copyright (C) 2012 Bernd Krumboeck (krumboeck@universalnet.at)
 *
 * This driver is inspired by the 3.2.0 version of drivers/net/can/usb/ems_usb.c
 * and drivers/net/can/usb/esd_usb2.c
 *
 * Many thanks to Gerhard Bertelsmann (info@gerhard-bertelsmann.de)
 * for testing and fixing this driver. Also many thanks to "8 devices",
 * who were very cooperative and answered my questions.
 */

#include <linux/ethtool.h>
#include <linux/signal.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/usb.h>

#include <linux/can.h>
#include <linux/can/dev.h>
#include <linux/can/error.h>

/* driver constants */
#define MAX_RX_URBS                        20
#define MAX_TX_URBS                        20
#define RX_BUFFER_SIZE                        64

/* vendor and product id */
#define USB_8DEV_VENDOR_ID                0x0483
#define USB_8DEV_PRODUCT_ID                0x1234

/* endpoints */
enum usb_8dev_endpoint {
        USB_8DEV_ENDP_DATA_RX = 1,
        USB_8DEV_ENDP_DATA_TX,
        USB_8DEV_ENDP_CMD_RX,
        USB_8DEV_ENDP_CMD_TX
};

/* device CAN clock */
#define USB_8DEV_ABP_CLOCK                32000000

/* setup flags */
#define USB_8DEV_SILENT                        0x01
#define USB_8DEV_LOOPBACK                0x02
#define USB_8DEV_DISABLE_AUTO_RESTRANS        0x04
#define USB_8DEV_STATUS_FRAME                0x08

/* commands */
enum usb_8dev_cmd {
        USB_8DEV_RESET = 1,
        USB_8DEV_OPEN,
        USB_8DEV_CLOSE,
        USB_8DEV_SET_SPEED,
        USB_8DEV_SET_MASK_FILTER,
        USB_8DEV_GET_STATUS,
        USB_8DEV_GET_STATISTICS,
        USB_8DEV_GET_SERIAL,
        USB_8DEV_GET_SOFTW_VER,
        USB_8DEV_GET_HARDW_VER,
        USB_8DEV_RESET_TIMESTAMP,
        USB_8DEV_GET_SOFTW_HARDW_VER
};

/* command options */
#define USB_8DEV_BAUD_MANUAL                0x09
#define USB_8DEV_CMD_START                0x11
#define USB_8DEV_CMD_END                0x22

#define USB_8DEV_CMD_SUCCESS                0
#define USB_8DEV_CMD_ERROR                255

#define USB_8DEV_CMD_TIMEOUT                1000

/* frames */
#define USB_8DEV_DATA_START                0x55
#define USB_8DEV_DATA_END                0xAA

#define USB_8DEV_TYPE_CAN_FRAME                0
#define USB_8DEV_TYPE_ERROR_FRAME        3

#define USB_8DEV_EXTID                        0x01
#define USB_8DEV_RTR                        0x02
#define USB_8DEV_ERR_FLAG                0x04

/* status */
#define USB_8DEV_STATUSMSG_OK                0x00  /* Normal condition. */
#define USB_8DEV_STATUSMSG_OVERRUN        0x01  /* Overrun occurred when sending */
#define USB_8DEV_STATUSMSG_BUSLIGHT        0x02  /* Error counter has reached 96 */
#define USB_8DEV_STATUSMSG_BUSHEAVY        0x03  /* Error count. has reached 128 */
#define USB_8DEV_STATUSMSG_BUSOFF        0x04  /* Device is in BUSOFF */
#define USB_8DEV_STATUSMSG_STUFF        0x20  /* Stuff Error */
#define USB_8DEV_STATUSMSG_FORM                0x21  /* Form Error */
#define USB_8DEV_STATUSMSG_ACK                0x23  /* Ack Error */
#define USB_8DEV_STATUSMSG_BIT0                0x24  /* Bit1 Error */
#define USB_8DEV_STATUSMSG_BIT1                0x25  /* Bit0 Error */
#define USB_8DEV_STATUSMSG_CRC                0x27  /* CRC Error */

#define USB_8DEV_RP_MASK                0x7F  /* Mask for Receive Error Bit */


/* table of devices that work with this driver */
static const struct usb_device_id usb_8dev_table[] = {
        { USB_DEVICE(USB_8DEV_VENDOR_ID, USB_8DEV_PRODUCT_ID) },
        { }                                        /* Terminating entry */
};

MODULE_DEVICE_TABLE(usb, usb_8dev_table);

struct usb_8dev_tx_urb_context {
        struct usb_8dev_priv *priv;

        u32 echo_index;
};

/* Structure to hold all of our device specific stuff */
struct usb_8dev_priv {
        struct can_priv can; /* must be the first member */

        struct usb_device *udev;
        struct net_device *netdev;

        atomic_t active_tx_urbs;
        struct usb_anchor tx_submitted;
        struct usb_8dev_tx_urb_context tx_contexts[MAX_TX_URBS];

        struct usb_anchor rx_submitted;

        struct can_berr_counter bec;

        u8 *cmd_msg_buffer;

        struct mutex usb_8dev_cmd_lock;
        void *rxbuf[MAX_RX_URBS];
        dma_addr_t rxbuf_dma[MAX_RX_URBS];
};

/* tx frame */
struct __packed usb_8dev_tx_msg {
        u8 begin;
        u8 flags;        /* RTR and EXT_ID flag */
        __be32 id;        /* upper 3 bits not used */
        u8 dlc;                /* data length code 0-8 bytes */
        u8 data[8];        /* 64-bit data */
        u8 end;
};

/* rx frame */
struct __packed usb_8dev_rx_msg {
        u8 begin;
        u8 type;                /* frame type */
        u8 flags;                /* RTR and EXT_ID flag */
        __be32 id;                /* upper 3 bits not used */
        u8 dlc;                        /* data length code 0-8 bytes */
        u8 data[8];                /* 64-bit data */
        __be32 timestamp;        /* 32-bit timestamp */
        u8 end;
};

/* command frame */
struct __packed usb_8dev_cmd_msg {
        u8 begin;
        u8 channel;        /* unknown - always 0 */
        u8 command;        /* command to execute */
        u8 opt1;        /* optional parameter / return value */
        u8 opt2;        /* optional parameter 2 */
        u8 data[10];        /* optional parameter and data */
        u8 end;
};

static int usb_8dev_send_cmd_msg(struct usb_8dev_priv *priv, u8 *msg, int size)
{
        int actual_length;

        return usb_bulk_msg(priv->udev,
                            usb_sndbulkpipe(priv->udev, USB_8DEV_ENDP_CMD_TX),
                            msg, size, &actual_length, USB_8DEV_CMD_TIMEOUT);
}

static int usb_8dev_wait_cmd_msg(struct usb_8dev_priv *priv, u8 *msg, int size,
                                int *actual_length)
{
        return usb_bulk_msg(priv->udev,
                            usb_rcvbulkpipe(priv->udev, USB_8DEV_ENDP_CMD_RX),
                            msg, size, actual_length, USB_8DEV_CMD_TIMEOUT);
}

/* Send command to device and receive result.
 * Command was successful when opt1 = 0.
 */
static int usb_8dev_send_cmd(struct usb_8dev_priv *priv,
                             struct usb_8dev_cmd_msg *out,
                             struct usb_8dev_cmd_msg *in)
{
        int err;
        int num_bytes_read;
        struct net_device *netdev;

        netdev = priv->netdev;

        out->begin = USB_8DEV_CMD_START;
        out->end = USB_8DEV_CMD_END;

        mutex_lock(&priv->usb_8dev_cmd_lock);

        memcpy(priv->cmd_msg_buffer, out,
                sizeof(struct usb_8dev_cmd_msg));

        err = usb_8dev_send_cmd_msg(priv, priv->cmd_msg_buffer,
                                    sizeof(struct usb_8dev_cmd_msg));
        if (err < 0) {
                netdev_err(netdev, "sending command message failed\n");
                goto failed;
        }

        err = usb_8dev_wait_cmd_msg(priv, priv->cmd_msg_buffer,
                                    sizeof(struct usb_8dev_cmd_msg),
                                    &num_bytes_read);
        if (err < 0) {
                netdev_err(netdev, "no command message answer\n");
                goto failed;
        }

        memcpy(in, priv->cmd_msg_buffer, sizeof(struct usb_8dev_cmd_msg));

        if (in->begin != USB_8DEV_CMD_START || in->end != USB_8DEV_CMD_END ||
                        num_bytes_read != 16 || in->opt1 != 0)
                err = -EPROTO;

failed:
        mutex_unlock(&priv->usb_8dev_cmd_lock);
        return err;
}

/* Send open command to device */
static int usb_8dev_cmd_open(struct usb_8dev_priv *priv)
{
        struct can_bittiming *bt = &priv->can.bittiming;
        struct usb_8dev_cmd_msg outmsg;
        struct usb_8dev_cmd_msg inmsg;
        u32 ctrlmode = priv->can.ctrlmode;
        u32 flags = USB_8DEV_STATUS_FRAME;
        __be32 beflags;
        __be16 bebrp;

        memset(&outmsg, 0, sizeof(outmsg));
        outmsg.command = USB_8DEV_OPEN;
        outmsg.opt1 = USB_8DEV_BAUD_MANUAL;
        outmsg.data[0] = bt->prop_seg + bt->phase_seg1;
        outmsg.data[1] = bt->phase_seg2;
        outmsg.data[2] = bt->sjw;

        /* BRP */
        bebrp = cpu_to_be16((u16)bt->brp);
        memcpy(&outmsg.data[3], &bebrp, sizeof(bebrp));

        /* flags */
        if (ctrlmode & CAN_CTRLMODE_LOOPBACK)
                flags |= USB_8DEV_LOOPBACK;
        if (ctrlmode & CAN_CTRLMODE_LISTENONLY)
                flags |= USB_8DEV_SILENT;
        if (ctrlmode & CAN_CTRLMODE_ONE_SHOT)
                flags |= USB_8DEV_DISABLE_AUTO_RESTRANS;

        beflags = cpu_to_be32(flags);
        memcpy(&outmsg.data[5], &beflags, sizeof(beflags));

        return usb_8dev_send_cmd(priv, &outmsg, &inmsg);
}

/* Send close command to device */
static int usb_8dev_cmd_close(struct usb_8dev_priv *priv)
{
        struct usb_8dev_cmd_msg inmsg;
        struct usb_8dev_cmd_msg outmsg = {
                .channel = 0,
                .command = USB_8DEV_CLOSE,
                .opt1 = 0,
                .opt2 = 0
        };

        return usb_8dev_send_cmd(priv, &outmsg, &inmsg);
}

/* Get firmware and hardware version */
static int usb_8dev_cmd_version(struct usb_8dev_priv *priv, u32 *res)
{
        struct usb_8dev_cmd_msg        inmsg;
        struct usb_8dev_cmd_msg        outmsg = {
                .channel = 0,
                .command = USB_8DEV_GET_SOFTW_HARDW_VER,
                .opt1 = 0,
                .opt2 = 0
        };

        int err = usb_8dev_send_cmd(priv, &outmsg, &inmsg);
        if (err)
                return err;

        *res = be32_to_cpup((__be32 *)inmsg.data);

        return err;
}

/* Set network device mode
 *
 * Maybe we should leave this function empty, because the device
 * set mode variable with open command.
 */
static int usb_8dev_set_mode(struct net_device *netdev, enum can_mode mode)
{
        struct usb_8dev_priv *priv = netdev_priv(netdev);
        int err = 0;

        switch (mode) {
        case CAN_MODE_START:
                err = usb_8dev_cmd_open(priv);
                if (err)
                        netdev_warn(netdev, "couldn't start device");
                break;

        default:
                return -EOPNOTSUPP;
        }

        return err;
}

/* Read error/status frames */
static void usb_8dev_rx_err_msg(struct usb_8dev_priv *priv,
                                struct usb_8dev_rx_msg *msg)
{
        struct can_frame *cf;
        struct sk_buff *skb;
        struct net_device_stats *stats = &priv->netdev->stats;

        /* Error message:
         * byte 0: Status
         * byte 1: bit   7: Receive Passive
         * byte 1: bit 0-6: Receive Error Counter
         * byte 2: Transmit Error Counter
         * byte 3: Always 0 (maybe reserved for future use)
         */

        u8 state = msg->data[0];
        u8 rxerr = msg->data[1] & USB_8DEV_RP_MASK;
        u8 txerr = msg->data[2];
        int rx_errors = 0;
        int tx_errors = 0;

        skb = alloc_can_err_skb(priv->netdev, &cf);
        if (!skb)
                return;

        switch (state) {
        case USB_8DEV_STATUSMSG_OK:
                priv->can.state = CAN_STATE_ERROR_ACTIVE;
                cf->can_id |= CAN_ERR_PROT;
                cf->data[2] = CAN_ERR_PROT_ACTIVE;
                break;
        case USB_8DEV_STATUSMSG_BUSOFF:
                priv->can.state = CAN_STATE_BUS_OFF;
                cf->can_id |= CAN_ERR_BUSOFF;
                priv->can.can_stats.bus_off++;
                can_bus_off(priv->netdev);
                break;
        case USB_8DEV_STATUSMSG_OVERRUN:
        case USB_8DEV_STATUSMSG_BUSLIGHT:
        case USB_8DEV_STATUSMSG_BUSHEAVY:
                cf->can_id |= CAN_ERR_CRTL;
                break;
        default:
                priv->can.state = CAN_STATE_ERROR_WARNING;
                cf->can_id |= CAN_ERR_PROT | CAN_ERR_BUSERROR;
                priv->can.can_stats.bus_error++;
                break;
        }

        switch (state) {
        case USB_8DEV_STATUSMSG_OK:
        case USB_8DEV_STATUSMSG_BUSOFF:
                break;
        case USB_8DEV_STATUSMSG_ACK:
                cf->can_id |= CAN_ERR_ACK;
                tx_errors = 1;
                break;
        case USB_8DEV_STATUSMSG_CRC:
                cf->data[3] = CAN_ERR_PROT_LOC_CRC_SEQ;
                rx_errors = 1;
                break;
        case USB_8DEV_STATUSMSG_BIT0:
                cf->data[2] |= CAN_ERR_PROT_BIT0;
                tx_errors = 1;
                break;
        case USB_8DEV_STATUSMSG_BIT1:
                cf->data[2] |= CAN_ERR_PROT_BIT1;
                tx_errors = 1;
                break;
        case USB_8DEV_STATUSMSG_FORM:
                cf->data[2] |= CAN_ERR_PROT_FORM;
                rx_errors = 1;
                break;
        case USB_8DEV_STATUSMSG_STUFF:
                cf->data[2] |= CAN_ERR_PROT_STUFF;
                rx_errors = 1;
                break;
        case USB_8DEV_STATUSMSG_OVERRUN:
                cf->data[1] = CAN_ERR_CRTL_RX_OVERFLOW;
                stats->rx_over_errors++;
                rx_errors = 1;
                break;
        case USB_8DEV_STATUSMSG_BUSLIGHT:
                priv->can.state = CAN_STATE_ERROR_WARNING;
                cf->data[1] = (txerr > rxerr) ?
                        CAN_ERR_CRTL_TX_WARNING :
                        CAN_ERR_CRTL_RX_WARNING;
                priv->can.can_stats.error_warning++;
                break;
        case USB_8DEV_STATUSMSG_BUSHEAVY:
                priv->can.state = CAN_STATE_ERROR_PASSIVE;
                cf->data[1] = (txerr > rxerr) ?
                        CAN_ERR_CRTL_TX_PASSIVE :
                        CAN_ERR_CRTL_RX_PASSIVE;
                priv->can.can_stats.error_passive++;
                break;
        default:
                netdev_warn(priv->netdev,
                            "Unknown status/error message (%d)\n", state);
                break;
        }

        if (tx_errors) {
                cf->data[2] |= CAN_ERR_PROT_TX;
                stats->tx_errors++;
        }

        if (rx_errors)
                stats->rx_errors++;
        if (priv->can.state != CAN_STATE_BUS_OFF) {
                cf->can_id |= CAN_ERR_CNT;
                cf->data[6] = txerr;
                cf->data[7] = rxerr;
        }

        priv->bec.txerr = txerr;
        priv->bec.rxerr = rxerr;

        netif_rx(skb);
}

/* Read data and status frames */
static void usb_8dev_rx_can_msg(struct usb_8dev_priv *priv,
                                struct usb_8dev_rx_msg *msg)
{
        struct can_frame *cf;
        struct sk_buff *skb;
        struct net_device_stats *stats = &priv->netdev->stats;

        if (msg->type == USB_8DEV_TYPE_ERROR_FRAME &&
                   msg->flags == USB_8DEV_ERR_FLAG) {
                usb_8dev_rx_err_msg(priv, msg);
        } else if (msg->type == USB_8DEV_TYPE_CAN_FRAME) {
                skb = alloc_can_skb(priv->netdev, &cf);
                if (!skb)
                        return;

                cf->can_id = be32_to_cpu(msg->id);
                can_frame_set_cc_len(cf, msg->dlc & 0xF, priv->can.ctrlmode);

                if (msg->flags & USB_8DEV_EXTID)
                        cf->can_id |= CAN_EFF_FLAG;

                if (msg->flags & USB_8DEV_RTR) {
                        cf->can_id |= CAN_RTR_FLAG;
                } else {
                        memcpy(cf->data, msg->data, cf->len);
                        stats->rx_bytes += cf->len;
                }
                stats->rx_packets++;

                netif_rx(skb);
        } else {
                netdev_warn(priv->netdev, "frame type %d unknown",
                         msg->type);
        }

}

/* Callback for reading data from device
 *
 * Check urb status, call read function and resubmit urb read operation.
 */
static void usb_8dev_read_bulk_callback(struct urb *urb)
{
        struct usb_8dev_priv *priv = urb->context;
        struct net_device *netdev;
        int retval;
        int pos = 0;

        netdev = priv->netdev;

        if (!netif_device_present(netdev))
                return;

        switch (urb->status) {
        case 0: /* success */
                break;

        case -ENOENT:
        case -EPIPE:
        case -EPROTO:
        case -ESHUTDOWN:
                return;

        default:
                netdev_info(netdev, "Rx URB aborted (%d)\n",
                         urb->status);
                goto resubmit_urb;
        }

        while (pos < urb->actual_length) {
                struct usb_8dev_rx_msg *msg;

                if (pos + sizeof(struct usb_8dev_rx_msg) > urb->actual_length) {
                        netdev_err(priv->netdev, "format error\n");
                        break;
                }

                msg = (struct usb_8dev_rx_msg *)(urb->transfer_buffer + pos);
                usb_8dev_rx_can_msg(priv, msg);

                pos += sizeof(struct usb_8dev_rx_msg);
        }

resubmit_urb:
        usb_fill_bulk_urb(urb, priv->udev,
                          usb_rcvbulkpipe(priv->udev, USB_8DEV_ENDP_DATA_RX),
                          urb->transfer_buffer, RX_BUFFER_SIZE,
                          usb_8dev_read_bulk_callback, priv);

        retval = usb_submit_urb(urb, GFP_ATOMIC);

        if (retval == -ENODEV)
                netif_device_detach(netdev);
        else if (retval)
                netdev_err(netdev,
                        "failed resubmitting read bulk urb: %d\n", retval);
}

/* Callback handler for write operations
 *
 * Free allocated buffers, check transmit status and
 * calculate statistic.
 */
static void usb_8dev_write_bulk_callback(struct urb *urb)
{
        struct usb_8dev_tx_urb_context *context = urb->context;
        struct usb_8dev_priv *priv;
        struct net_device *netdev;

        BUG_ON(!context);

        priv = context->priv;
        netdev = priv->netdev;

        /* free up our allocated buffer */
        usb_free_coherent(urb->dev, urb->transfer_buffer_length,
                          urb->transfer_buffer, urb->transfer_dma);

        atomic_dec(&priv->active_tx_urbs);

        if (!netif_device_present(netdev))
                return;

        if (urb->status)
                netdev_info(netdev, "Tx URB aborted (%d)\n",
                         urb->status);

        netdev->stats.tx_packets++;
        netdev->stats.tx_bytes += can_get_echo_skb(netdev, context->echo_index, NULL);

        /* Release context */
        context->echo_index = MAX_TX_URBS;

        netif_wake_queue(netdev);
}

/* Send data to device */
static netdev_tx_t usb_8dev_start_xmit(struct sk_buff *skb,
                                      struct net_device *netdev)
{
        struct usb_8dev_priv *priv = netdev_priv(netdev);
        struct net_device_stats *stats = &netdev->stats;
        struct can_frame *cf = (struct can_frame *) skb->data;
        struct usb_8dev_tx_msg *msg;
        struct urb *urb;
        struct usb_8dev_tx_urb_context *context = NULL;
        u8 *buf;
        int i, err;
        size_t size = sizeof(struct usb_8dev_tx_msg);

        if (can_dev_dropped_skb(netdev, skb))
                return NETDEV_TX_OK;

        /* create a URB, and a buffer for it, and copy the data to the URB */
        urb = usb_alloc_urb(0, GFP_ATOMIC);
        if (!urb)
                goto nomem;

        buf = usb_alloc_coherent(priv->udev, size, GFP_ATOMIC,
                                 &urb->transfer_dma);
        if (!buf) {
                netdev_err(netdev, "No memory left for USB buffer\n");
                goto nomembuf;
        }

        memset(buf, 0, size);

        msg = (struct usb_8dev_tx_msg *)buf;
        msg->begin = USB_8DEV_DATA_START;
        msg->flags = 0x00;

        if (cf->can_id & CAN_RTR_FLAG)
                msg->flags |= USB_8DEV_RTR;

        if (cf->can_id & CAN_EFF_FLAG)
                msg->flags |= USB_8DEV_EXTID;

        msg->id = cpu_to_be32(cf->can_id & CAN_ERR_MASK);
        msg->dlc = can_get_cc_dlc(cf, priv->can.ctrlmode);
        memcpy(msg->data, cf->data, cf->len);
        msg->end = USB_8DEV_DATA_END;

        for (i = 0; i < MAX_TX_URBS; i++) {
                if (priv->tx_contexts[i].echo_index == MAX_TX_URBS) {
                        context = &priv->tx_contexts[i];
                        break;
                }
        }

        /* May never happen! When this happens we'd more URBs in flight as
         * allowed (MAX_TX_URBS).
         */
        if (!context)
                goto nofreecontext;

        context->priv = priv;
        context->echo_index = i;

        usb_fill_bulk_urb(urb, priv->udev,
                          usb_sndbulkpipe(priv->udev, USB_8DEV_ENDP_DATA_TX),
                          buf, size, usb_8dev_write_bulk_callback, context);
        urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
        usb_anchor_urb(urb, &priv->tx_submitted);

        can_put_echo_skb(skb, netdev, context->echo_index, 0);

        atomic_inc(&priv->active_tx_urbs);

        err = usb_submit_urb(urb, GFP_ATOMIC);
        if (unlikely(err)) {
                can_free_echo_skb(netdev, context->echo_index, NULL);

                usb_unanchor_urb(urb);
                usb_free_coherent(priv->udev, size, buf, urb->transfer_dma);

                atomic_dec(&priv->active_tx_urbs);

                if (err == -ENODEV)
                        netif_device_detach(netdev);
                else
                        netdev_warn(netdev, "failed tx_urb %d\n", err);
                stats->tx_dropped++;
        } else if (atomic_read(&priv->active_tx_urbs) >= MAX_TX_URBS)
                /* Slow down tx path */
                netif_stop_queue(netdev);

        /* Release our reference to this URB, the USB core will eventually free
         * it entirely.
         */
        usb_free_urb(urb);

        return NETDEV_TX_OK;

nofreecontext:
        usb_free_coherent(priv->udev, size, buf, urb->transfer_dma);
        usb_free_urb(urb);

        netdev_warn(netdev, "couldn't find free context");

        return NETDEV_TX_BUSY;

nomembuf:
        usb_free_urb(urb);

nomem:
        dev_kfree_skb(skb);
        stats->tx_dropped++;

        return NETDEV_TX_OK;
}

static int usb_8dev_get_berr_counter(const struct net_device *netdev,
                                     struct can_berr_counter *bec)
{
        struct usb_8dev_priv *priv = netdev_priv(netdev);

        bec->txerr = priv->bec.txerr;
        bec->rxerr = priv->bec.rxerr;

        return 0;
}

/* Start USB device */
static int usb_8dev_start(struct usb_8dev_priv *priv)
{
        struct net_device *netdev = priv->netdev;
        int err, i;

        for (i = 0; i < MAX_RX_URBS; i++) {
                struct urb *urb = NULL;
                u8 *buf;
                dma_addr_t buf_dma;

                /* create a URB, and a buffer for it */
                urb = usb_alloc_urb(0, GFP_KERNEL);
                if (!urb) {
                        err = -ENOMEM;
                        break;
                }

                buf = usb_alloc_coherent(priv->udev, RX_BUFFER_SIZE, GFP_KERNEL,
                                         &buf_dma);
                if (!buf) {
                        netdev_err(netdev, "No memory left for USB buffer\n");
                        usb_free_urb(urb);
                        err = -ENOMEM;
                        break;
                }

                urb->transfer_dma = buf_dma;

                usb_fill_bulk_urb(urb, priv->udev,
                                  usb_rcvbulkpipe(priv->udev,
                                                  USB_8DEV_ENDP_DATA_RX),
                                  buf, RX_BUFFER_SIZE,
                                  usb_8dev_read_bulk_callback, priv);
                urb->transfer_flags |= URB_NO_TRANSFER_DMA_MAP;
                usb_anchor_urb(urb, &priv->rx_submitted);

                err = usb_submit_urb(urb, GFP_KERNEL);
                if (err) {
                        usb_unanchor_urb(urb);
                        usb_free_coherent(priv->udev, RX_BUFFER_SIZE, buf,
                                          urb->transfer_dma);
                        usb_free_urb(urb);
                        break;
                }

                priv->rxbuf[i] = buf;
                priv->rxbuf_dma[i] = buf_dma;

                /* Drop reference, USB core will take care of freeing it */
                usb_free_urb(urb);
        }

        /* Did we submit any URBs */
        if (i == 0) {
                netdev_warn(netdev, "couldn't setup read URBs\n");
                return err;
        }

        /* Warn if we've couldn't transmit all the URBs */
        if (i < MAX_RX_URBS)
                netdev_warn(netdev, "rx performance may be slow\n");

        err = usb_8dev_cmd_open(priv);
        if (err)
                goto failed;

        priv->can.state = CAN_STATE_ERROR_ACTIVE;

        return 0;

failed:
        if (err == -ENODEV)
                netif_device_detach(priv->netdev);

        netdev_warn(netdev, "couldn't submit control: %d\n", err);

        return err;
}

/* Open USB device */
static int usb_8dev_open(struct net_device *netdev)
{
        struct usb_8dev_priv *priv = netdev_priv(netdev);
        int err;

        /* common open */
        err = open_candev(netdev);
        if (err)
                return err;

        /* finally start device */
        err = usb_8dev_start(priv);
        if (err) {
                if (err == -ENODEV)
                        netif_device_detach(priv->netdev);

                netdev_warn(netdev, "couldn't start device: %d\n",
                         err);

                close_candev(netdev);

                return err;
        }

        netif_start_queue(netdev);

        return 0;
}

static void unlink_all_urbs(struct usb_8dev_priv *priv)
{
        int i;

        usb_kill_anchored_urbs(&priv->rx_submitted);

        for (i = 0; i < MAX_RX_URBS; ++i)
                usb_free_coherent(priv->udev, RX_BUFFER_SIZE,
                                  priv->rxbuf[i], priv->rxbuf_dma[i]);

        usb_kill_anchored_urbs(&priv->tx_submitted);
        atomic_set(&priv->active_tx_urbs, 0);

        for (i = 0; i < MAX_TX_URBS; i++)
                priv->tx_contexts[i].echo_index = MAX_TX_URBS;
}

/* Close USB device */
static int usb_8dev_close(struct net_device *netdev)
{
        struct usb_8dev_priv *priv = netdev_priv(netdev);
        int err = 0;

        /* Send CLOSE command to CAN controller */
        err = usb_8dev_cmd_close(priv);
        if (err)
                netdev_warn(netdev, "couldn't stop device");

        priv->can.state = CAN_STATE_STOPPED;

        netif_stop_queue(netdev);

        /* Stop polling */
        unlink_all_urbs(priv);

        close_candev(netdev);

        return err;
}

static const struct net_device_ops usb_8dev_netdev_ops = {
        .ndo_open = usb_8dev_open,
        .ndo_stop = usb_8dev_close,
        .ndo_start_xmit = usb_8dev_start_xmit,
        .ndo_change_mtu = can_change_mtu,
};

static const struct ethtool_ops usb_8dev_ethtool_ops = {
        .get_ts_info = ethtool_op_get_ts_info,
};

static const struct can_bittiming_const usb_8dev_bittiming_const = {
        .name = KBUILD_MODNAME,
        .tseg1_min = 1,
        .tseg1_max = 16,
        .tseg2_min = 1,
        .tseg2_max = 8,
        .sjw_max = 4,
        .brp_min = 1,
        .brp_max = 1024,
        .brp_inc = 1,
};

/* Probe USB device
 *
 * Check device and firmware.
 * Set supported modes and bittiming constants.
 * Allocate some memory.
 */
static int usb_8dev_probe(struct usb_interface *intf,
                         const struct usb_device_id *id)
{
        struct net_device *netdev;
        struct usb_8dev_priv *priv;
        int i, err = -ENOMEM;
        u32 version;
        char buf[18];
        struct usb_device *usbdev = interface_to_usbdev(intf);

        /* product id looks strange, better we also check iProduct string */
        if (usb_string(usbdev, usbdev->descriptor.iProduct, buf,
                       sizeof(buf)) > 0 && strcmp(buf, "USB2CAN converter")) {
                dev_info(&usbdev->dev, "ignoring: not an USB2CAN converter\n");
                return -ENODEV;
        }

        netdev = alloc_candev(sizeof(struct usb_8dev_priv), MAX_TX_URBS);
        if (!netdev) {
                dev_err(&intf->dev, "Couldn't alloc candev\n");
                return -ENOMEM;
        }

        priv = netdev_priv(netdev);

        priv->udev = usbdev;
        priv->netdev = netdev;

        priv->can.state = CAN_STATE_STOPPED;
        priv->can.clock.freq = USB_8DEV_ABP_CLOCK;
        priv->can.bittiming_const = &usb_8dev_bittiming_const;
        priv->can.do_set_mode = usb_8dev_set_mode;
        priv->can.do_get_berr_counter = usb_8dev_get_berr_counter;
        priv->can.ctrlmode_supported = CAN_CTRLMODE_LOOPBACK |
                                      CAN_CTRLMODE_LISTENONLY |
                                      CAN_CTRLMODE_ONE_SHOT |
                                      CAN_CTRLMODE_CC_LEN8_DLC;

        netdev->netdev_ops = &usb_8dev_netdev_ops;
        netdev->ethtool_ops = &usb_8dev_ethtool_ops;

        netdev->flags |= IFF_ECHO; /* we support local echo */

        init_usb_anchor(&priv->rx_submitted);

        init_usb_anchor(&priv->tx_submitted);
        atomic_set(&priv->active_tx_urbs, 0);

        for (i = 0; i < MAX_TX_URBS; i++)
                priv->tx_contexts[i].echo_index = MAX_TX_URBS;

        priv->cmd_msg_buffer = devm_kzalloc(&intf->dev, sizeof(struct usb_8dev_cmd_msg),
                                            GFP_KERNEL);
        if (!priv->cmd_msg_buffer)
                goto cleanup_candev;

        usb_set_intfdata(intf, priv);

        SET_NETDEV_DEV(netdev, &intf->dev);

        mutex_init(&priv->usb_8dev_cmd_lock);

        err = register_candev(netdev);
        if (err) {
                netdev_err(netdev,
                        "couldn't register CAN device: %d\n", err);
                goto cleanup_candev;
        }

        err = usb_8dev_cmd_version(priv, &version);
        if (err) {
                netdev_err(netdev, "can't get firmware version\n");
                goto cleanup_unregister_candev;
        } else {
                netdev_info(netdev,
                         "firmware: %d.%d, hardware: %d.%d\n",
                         (version>>24) & 0xff, (version>>16) & 0xff,
                         (version>>8) & 0xff, version & 0xff);
        }

        return 0;

cleanup_unregister_candev:
        unregister_netdev(priv->netdev);

cleanup_candev:
        free_candev(netdev);

        return err;

}

/* Called by the usb core when driver is unloaded or device is removed */
static void usb_8dev_disconnect(struct usb_interface *intf)
{
        struct usb_8dev_priv *priv = usb_get_intfdata(intf);

        usb_set_intfdata(intf, NULL);

        if (priv) {
                netdev_info(priv->netdev, "device disconnected\n");

                unregister_netdev(priv->netdev);
                unlink_all_urbs(priv);
                free_candev(priv->netdev);
        }

}

static struct usb_driver usb_8dev_driver = {
        .name =                KBUILD_MODNAME,
        .probe =        usb_8dev_probe,
        .disconnect =        usb_8dev_disconnect,
        .id_table =        usb_8dev_table,
};

module_usb_driver(usb_8dev_driver);

MODULE_AUTHOR("Bernd Krumboeck <krumboeck@universalnet.at>");
MODULE_DESCRIPTION("CAN driver for 8 devices USB2CAN interfaces");
MODULE_LICENSE("GPL v2");













































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    4 






















































































    6 






    6 












    4 


    4 

    4 

    4 






    4 






    4 






    4 






    4 
































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































    2 




    2 












    2 





   13 
   13 













































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
// SPDX-License-Identifier: GPL-2.0

#include <linux/ceph/ceph_debug.h>

#include <linux/module.h>
#include <linux/slab.h>

#include <linux/ceph/libceph.h>
#include <linux/ceph/osdmap.h>
#include <linux/ceph/decode.h>
#include <linux/crush/hash.h>
#include <linux/crush/mapper.h>

static __printf(2, 3)
void osdmap_info(const struct ceph_osdmap *map, const char *fmt, ...)
{
        struct va_format vaf;
        va_list args;

        va_start(args, fmt);
        vaf.fmt = fmt;
        vaf.va = &args;

        printk(KERN_INFO "%s (%pU e%u): %pV", KBUILD_MODNAME, &map->fsid,
               map->epoch, &vaf);

        va_end(args);
}

char *ceph_osdmap_state_str(char *str, int len, u32 state)
{
        if (!len)
                return str;

        if ((state & CEPH_OSD_EXISTS) && (state & CEPH_OSD_UP))
                snprintf(str, len, "exists, up");
        else if (state & CEPH_OSD_EXISTS)
                snprintf(str, len, "exists");
        else if (state & CEPH_OSD_UP)
                snprintf(str, len, "up");
        else
                snprintf(str, len, "doesn't exist");

        return str;
}

/* maps */

static int calc_bits_of(unsigned int t)
{
        int b = 0;
        while (t) {
                t = t >> 1;
                b++;
        }
        return b;
}

/*
 * the foo_mask is the smallest value 2^n-1 that is >= foo.
 */
static void calc_pg_masks(struct ceph_pg_pool_info *pi)
{
        pi->pg_num_mask = (1 << calc_bits_of(pi->pg_num-1)) - 1;
        pi->pgp_num_mask = (1 << calc_bits_of(pi->pgp_num-1)) - 1;
}

/*
 * decode crush map
 */
static int crush_decode_uniform_bucket(void **p, void *end,
                                       struct crush_bucket_uniform *b)
{
        dout("crush_decode_uniform_bucket %p to %p\n", *p, end);
        ceph_decode_need(p, end, (1+b->h.size) * sizeof(u32), bad);
        b->item_weight = ceph_decode_32(p);
        return 0;
bad:
        return -EINVAL;
}

static int crush_decode_list_bucket(void **p, void *end,
                                    struct crush_bucket_list *b)
{
        int j;
        dout("crush_decode_list_bucket %p to %p\n", *p, end);
        b->item_weights = kcalloc(b->h.size, sizeof(u32), GFP_NOFS);
        if (b->item_weights == NULL)
                return -ENOMEM;
        b->sum_weights = kcalloc(b->h.size, sizeof(u32), GFP_NOFS);
        if (b->sum_weights == NULL)
                return -ENOMEM;
        ceph_decode_need(p, end, 2 * b->h.size * sizeof(u32), bad);
        for (j = 0; j < b->h.size; j++) {
                b->item_weights[j] = ceph_decode_32(p);
                b->sum_weights[j] = ceph_decode_32(p);
        }
        return 0;
bad:
        return -EINVAL;
}

static int crush_decode_tree_bucket(void **p, void *end,
                                    struct crush_bucket_tree *b)
{
        int j;
        dout("crush_decode_tree_bucket %p to %p\n", *p, end);
        ceph_decode_8_safe(p, end, b->num_nodes, bad);
        b->node_weights = kcalloc(b->num_nodes, sizeof(u32), GFP_NOFS);
        if (b->node_weights == NULL)
                return -ENOMEM;
        ceph_decode_need(p, end, b->num_nodes * sizeof(u32), bad);
        for (j = 0; j < b->num_nodes; j++)
                b->node_weights[j] = ceph_decode_32(p);
        return 0;
bad:
        return -EINVAL;
}

static int crush_decode_straw_bucket(void **p, void *end,
                                     struct crush_bucket_straw *b)
{
        int j;
        dout("crush_decode_straw_bucket %p to %p\n", *p, end);
        b->item_weights = kcalloc(b->h.size, sizeof(u32), GFP_NOFS);
        if (b->item_weights == NULL)
                return -ENOMEM;
        b->straws = kcalloc(b->h.size, sizeof(u32), GFP_NOFS);
        if (b->straws == NULL)
                return -ENOMEM;
        ceph_decode_need(p, end, 2 * b->h.size * sizeof(u32), bad);
        for (j = 0; j < b->h.size; j++) {
                b->item_weights[j] = ceph_decode_32(p);
                b->straws[j] = ceph_decode_32(p);
        }
        return 0;
bad:
        return -EINVAL;
}

static int crush_decode_straw2_bucket(void **p, void *end,
                                      struct crush_bucket_straw2 *b)
{
        int j;
        dout("crush_decode_straw2_bucket %p to %p\n", *p, end);
        b->item_weights = kcalloc(b->h.size, sizeof(u32), GFP_NOFS);
        if (b->item_weights == NULL)
                return -ENOMEM;
        ceph_decode_need(p, end, b->h.size * sizeof(u32), bad);
        for (j = 0; j < b->h.size; j++)
                b->item_weights[j] = ceph_decode_32(p);
        return 0;
bad:
        return -EINVAL;
}

struct crush_name_node {
        struct rb_node cn_node;
        int cn_id;
        char cn_name[];
};

static struct crush_name_node *alloc_crush_name(size_t name_len)
{
        struct crush_name_node *cn;

        cn = kmalloc(sizeof(*cn) + name_len + 1, GFP_NOIO);
        if (!cn)
                return NULL;

        RB_CLEAR_NODE(&cn->cn_node);
        return cn;
}

static void free_crush_name(struct crush_name_node *cn)
{
        WARN_ON(!RB_EMPTY_NODE(&cn->cn_node));

        kfree(cn);
}

DEFINE_RB_FUNCS(crush_name, struct crush_name_node, cn_id, cn_node)

static int decode_crush_names(void **p, void *end, struct rb_root *root)
{
        u32 n;

        ceph_decode_32_safe(p, end, n, e_inval);
        while (n--) {
                struct crush_name_node *cn;
                int id;
                u32 name_len;

                ceph_decode_32_safe(p, end, id, e_inval);
                ceph_decode_32_safe(p, end, name_len, e_inval);
                ceph_decode_need(p, end, name_len, e_inval);

                cn = alloc_crush_name(name_len);
                if (!cn)
                        return -ENOMEM;

                cn->cn_id = id;
                memcpy(cn->cn_name, *p, name_len);
                cn->cn_name[name_len] = '\0';
                *p += name_len;

                if (!__insert_crush_name(root, cn)) {
                        free_crush_name(cn);
                        return -EEXIST;
                }
        }

        return 0;

e_inval:
        return -EINVAL;
}

void clear_crush_names(struct rb_root *root)
{
        while (!RB_EMPTY_ROOT(root)) {
                struct crush_name_node *cn =
                    rb_entry(rb_first(root), struct crush_name_node, cn_node);

                erase_crush_name(root, cn);
                free_crush_name(cn);
        }
}

static struct crush_choose_arg_map *alloc_choose_arg_map(void)
{
        struct crush_choose_arg_map *arg_map;

        arg_map = kzalloc(sizeof(*arg_map), GFP_NOIO);
        if (!arg_map)
                return NULL;

        RB_CLEAR_NODE(&arg_map->node);
        return arg_map;
}

static void free_choose_arg_map(struct crush_choose_arg_map *arg_map)
{
        if (arg_map) {
                int i, j;

                WARN_ON(!RB_EMPTY_NODE(&arg_map->node));

                for (i = 0; i < arg_map->size; i++) {
                        struct crush_choose_arg *arg = &arg_map->args[i];

                        for (j = 0; j < arg->weight_set_size; j++)
                                kfree(arg->weight_set[j].weights);
                        kfree(arg->weight_set);
                        kfree(arg->ids);
                }
                kfree(arg_map->args);
                kfree(arg_map);
        }
}

DEFINE_RB_FUNCS(choose_arg_map, struct crush_choose_arg_map, choose_args_index,
                node);

void clear_choose_args(struct crush_map *c)
{
        while (!RB_EMPTY_ROOT(&c->choose_args)) {
                struct crush_choose_arg_map *arg_map =
                    rb_entry(rb_first(&c->choose_args),
                             struct crush_choose_arg_map, node);

                erase_choose_arg_map(&c->choose_args, arg_map);
                free_choose_arg_map(arg_map);
        }
}

static u32 *decode_array_32_alloc(void **p, void *end, u32 *plen)
{
        u32 *a = NULL;
        u32 len;
        int ret;

        ceph_decode_32_safe(p, end, len, e_inval);
        if (len) {
                u32 i;

                a = kmalloc_array(len, sizeof(u32), GFP_NOIO);
                if (!a) {
                        ret = -ENOMEM;
                        goto fail;
                }

                ceph_decode_need(p, end, len * sizeof(u32), e_inval);
                for (i = 0; i < len; i++)
                        a[i] = ceph_decode_32(p);
        }

        *plen = len;
        return a;

e_inval:
        ret = -EINVAL;
fail:
        kfree(a);
        return ERR_PTR(ret);
}

/*
 * Assumes @arg is zero-initialized.
 */
static int decode_choose_arg(void **p, void *end, struct crush_choose_arg *arg)
{
        int ret;

        ceph_decode_32_safe(p, end, arg->weight_set_size, e_inval);
        if (arg->weight_set_size) {
                u32 i;

                arg->weight_set = kmalloc_array(arg->weight_set_size,
                                                sizeof(*arg->weight_set),
                                                GFP_NOIO);
                if (!arg->weight_set)
                        return -ENOMEM;

                for (i = 0; i < arg->weight_set_size; i++) {
                        struct crush_weight_set *w = &arg->weight_set[i];

                        w->weights = decode_array_32_alloc(p, end, &w->size);
                        if (IS_ERR(w->weights)) {
                                ret = PTR_ERR(w->weights);
                                w->weights = NULL;
                                return ret;
                        }
                }
        }

        arg->ids = decode_array_32_alloc(p, end, &arg->ids_size);
        if (IS_ERR(arg->ids)) {
                ret = PTR_ERR(arg->ids);
                arg->ids = NULL;
                return ret;
        }

        return 0;

e_inval:
        return -EINVAL;
}

static int decode_choose_args(void **p, void *end, struct crush_map *c)
{
        struct crush_choose_arg_map *arg_map = NULL;
        u32 num_choose_arg_maps, num_buckets;
        int ret;

        ceph_decode_32_safe(p, end, num_choose_arg_maps, e_inval);
        while (num_choose_arg_maps--) {
                arg_map = alloc_choose_arg_map();
                if (!arg_map) {
                        ret = -ENOMEM;
                        goto fail;
                }

                ceph_decode_64_safe(p, end, arg_map->choose_args_index,
                                    e_inval);
                arg_map->size = c->max_buckets;
                arg_map->args = kcalloc(arg_map->size, sizeof(*arg_map->args),
                                        GFP_NOIO);
                if (!arg_map->args) {
                        ret = -ENOMEM;
                        goto fail;
                }

                ceph_decode_32_safe(p, end, num_buckets, e_inval);
                while (num_buckets--) {
                        struct crush_choose_arg *arg;
                        u32 bucket_index;

                        ceph_decode_32_safe(p, end, bucket_index, e_inval);
                        if (bucket_index >= arg_map->size)
                                goto e_inval;

                        arg = &arg_map->args[bucket_index];
                        ret = decode_choose_arg(p, end, arg);
                        if (ret)
                                goto fail;

                        if (arg->ids_size &&
                            arg->ids_size != c->buckets[bucket_index]->size)
                                goto e_inval;
                }

                insert_choose_arg_map(&c->choose_args, arg_map);
        }

        return 0;

e_inval:
        ret = -EINVAL;
fail:
        free_choose_arg_map(arg_map);
        return ret;
}

static void crush_finalize(struct crush_map *c)
{
        __s32 b;

        /* Space for the array of pointers to per-bucket workspace */
        c->working_size = sizeof(struct crush_work) +
            c->max_buckets * sizeof(struct crush_work_bucket *);

        for (b = 0; b < c->max_buckets; b++) {
                if (!c->buckets[b])
                        continue;

                switch (c->buckets[b]->alg) {
                default:
                        /*
                         * The base case, permutation variables and
                         * the pointer to the permutation array.
                         */
                        c->working_size += sizeof(struct crush_work_bucket);
                        break;
                }
                /* Every bucket has a permutation array. */
                c->working_size += c->buckets[b]->size * sizeof(__u32);
        }
}

static struct crush_map *crush_decode(void *pbyval, void *end)
{
        struct crush_map *c;
        int err;
        int i, j;
        void **p = &pbyval;
        void *start = pbyval;
        u32 magic;

        dout("crush_decode %p to %p len %d\n", *p, end, (int)(end - *p));

        c = kzalloc(sizeof(*c), GFP_NOFS);
        if (c == NULL)
                return ERR_PTR(-ENOMEM);

        c->type_names = RB_ROOT;
        c->names = RB_ROOT;
        c->choose_args = RB_ROOT;

        /* set tunables to default values */
        c->choose_local_tries = 2;
        c->choose_local_fallback_tries = 5;
        c->choose_total_tries = 19;
        c->chooseleaf_descend_once = 0;

        ceph_decode_need(p, end, 4*sizeof(u32), bad);
        magic = ceph_decode_32(p);
        if (magic != CRUSH_MAGIC) {
                pr_err("crush_decode magic %x != current %x\n",
                       (unsigned int)magic, (unsigned int)CRUSH_MAGIC);
                goto bad;
        }
        c->max_buckets = ceph_decode_32(p);
        c->max_rules = ceph_decode_32(p);
        c->max_devices = ceph_decode_32(p);

        c->buckets = kcalloc(c->max_buckets, sizeof(*c->buckets), GFP_NOFS);
        if (c->buckets == NULL)
                goto badmem;
        c->rules = kcalloc(c->max_rules, sizeof(*c->rules), GFP_NOFS);
        if (c->rules == NULL)
                goto badmem;

        /* buckets */
        for (i = 0; i < c->max_buckets; i++) {
                int size = 0;
                u32 alg;
                struct crush_bucket *b;

                ceph_decode_32_safe(p, end, alg, bad);
                if (alg == 0) {
                        c->buckets[i] = NULL;
                        continue;
                }
                dout("crush_decode bucket %d off %x %p to %p\n",
                     i, (int)(*p-start), *p, end);

                switch (alg) {
                case CRUSH_BUCKET_UNIFORM:
                        size = sizeof(struct crush_bucket_uniform);
                        break;
                case CRUSH_BUCKET_LIST:
                        size = sizeof(struct crush_bucket_list);
                        break;
                case CRUSH_BUCKET_TREE:
                        size = sizeof(struct crush_bucket_tree);
                        break;
                case CRUSH_BUCKET_STRAW:
                        size = sizeof(struct crush_bucket_straw);
                        break;
                case CRUSH_BUCKET_STRAW2:
                        size = sizeof(struct crush_bucket_straw2);
                        break;
                default:
                        goto bad;
                }
                BUG_ON(size == 0);
                b = c->buckets[i] = kzalloc(size, GFP_NOFS);
                if (b == NULL)
                        goto badmem;

                ceph_decode_need(p, end, 4*sizeof(u32), bad);
                b->id = ceph_decode_32(p);
                b->type = ceph_decode_16(p);
                b->alg = ceph_decode_8(p);
                b->hash = ceph_decode_8(p);
                b->weight = ceph_decode_32(p);
                b->size = ceph_decode_32(p);

                dout("crush_decode bucket size %d off %x %p to %p\n",
                     b->size, (int)(*p-start), *p, end);

                b->items = kcalloc(b->size, sizeof(__s32), GFP_NOFS);
                if (b->items == NULL)
                        goto badmem;

                ceph_decode_need(p, end, b->size*sizeof(u32), bad);
                for (j = 0; j < b->size; j++)
                        b->items[j] = ceph_decode_32(p);

                switch (b->alg) {
                case CRUSH_BUCKET_UNIFORM:
                        err = crush_decode_uniform_bucket(p, end,
                                  (struct crush_bucket_uniform *)b);
                        if (err < 0)
                                goto fail;
                        break;
                case CRUSH_BUCKET_LIST:
                        err = crush_decode_list_bucket(p, end,
                               (struct crush_bucket_list *)b);
                        if (err < 0)
                                goto fail;
                        break;
                case CRUSH_BUCKET_TREE:
                        err = crush_decode_tree_bucket(p, end,
                                (struct crush_bucket_tree *)b);
                        if (err < 0)
                                goto fail;
                        break;
                case CRUSH_BUCKET_STRAW:
                        err = crush_decode_straw_bucket(p, end,
                                (struct crush_bucket_straw *)b);
                        if (err < 0)
                                goto fail;
                        break;
                case CRUSH_BUCKET_STRAW2:
                        err = crush_decode_straw2_bucket(p, end,
                                (struct crush_bucket_straw2 *)b);
                        if (err < 0)
                                goto fail;
                        break;
                }
        }

        /* rules */
        dout("rule vec is %p\n", c->rules);
        for (i = 0; i < c->max_rules; i++) {
                u32 yes;
                struct crush_rule *r;

                ceph_decode_32_safe(p, end, yes, bad);
                if (!yes) {
                        dout("crush_decode NO rule %d off %x %p to %p\n",
                             i, (int)(*p-start), *p, end);
                        c->rules[i] = NULL;
                        continue;
                }

                dout("crush_decode rule %d off %x %p to %p\n",
                     i, (int)(*p-start), *p, end);

                /* len */
                ceph_decode_32_safe(p, end, yes, bad);
#if BITS_PER_LONG == 32
                if (yes > (ULONG_MAX - sizeof(*r))
                          / sizeof(struct crush_rule_step))
                        goto bad;
#endif
                r = kmalloc(struct_size(r, steps, yes), GFP_NOFS);
                if (r == NULL)
                        goto badmem;
                dout(" rule %d is at %p\n", i, r);
                c->rules[i] = r;
                r->len = yes;
                ceph_decode_copy_safe(p, end, &r->mask, 4, bad); /* 4 u8's */
                ceph_decode_need(p, end, r->len*3*sizeof(u32), bad);
                for (j = 0; j < r->len; j++) {
                        r->steps[j].op = ceph_decode_32(p);
                        r->steps[j].arg1 = ceph_decode_32(p);
                        r->steps[j].arg2 = ceph_decode_32(p);
                }
        }

        err = decode_crush_names(p, end, &c->type_names);
        if (err)
                goto fail;

        err = decode_crush_names(p, end, &c->names);
        if (err)
                goto fail;

        ceph_decode_skip_map(p, end, 32, string, bad); /* rule_name_map */

        /* tunables */
        ceph_decode_need(p, end, 3*sizeof(u32), done);
        c->choose_local_tries = ceph_decode_32(p);
        c->choose_local_fallback_tries =  ceph_decode_32(p);
        c->choose_total_tries = ceph_decode_32(p);
        dout("crush decode tunable choose_local_tries = %d\n",
             c->choose_local_tries);
        dout("crush decode tunable choose_local_fallback_tries = %d\n",
             c->choose_local_fallback_tries);
        dout("crush decode tunable choose_total_tries = %d\n",
             c->choose_total_tries);

        ceph_decode_need(p, end, sizeof(u32), done);
        c->chooseleaf_descend_once = ceph_decode_32(p);
        dout("crush decode tunable chooseleaf_descend_once = %d\n",
             c->chooseleaf_descend_once);

        ceph_decode_need(p, end, sizeof(u8), done);
        c->chooseleaf_vary_r = ceph_decode_8(p);
        dout("crush decode tunable chooseleaf_vary_r = %d\n",
             c->chooseleaf_vary_r);

        /* skip straw_calc_version, allowed_bucket_algs */
        ceph_decode_need(p, end, sizeof(u8) + sizeof(u32), done);
        *p += sizeof(u8) + sizeof(u32);

        ceph_decode_need(p, end, sizeof(u8), done);
        c->chooseleaf_stable = ceph_decode_8(p);
        dout("crush decode tunable chooseleaf_stable = %d\n",
             c->chooseleaf_stable);

        if (*p != end) {
                /* class_map */
                ceph_decode_skip_map(p, end, 32, 32, bad);
                /* class_name */
                ceph_decode_skip_map(p, end, 32, string, bad);
                /* class_bucket */
                ceph_decode_skip_map_of_map(p, end, 32, 32, 32, bad);
        }

        if (*p != end) {
                err = decode_choose_args(p, end, c);
                if (err)
                        goto fail;
        }

done:
        crush_finalize(c);
        dout("crush_decode success\n");
        return c;

badmem:
        err = -ENOMEM;
fail:
        dout("crush_decode fail %d\n", err);
        crush_destroy(c);
        return ERR_PTR(err);

bad:
        err = -EINVAL;
        goto fail;
}

int ceph_pg_compare(const struct ceph_pg *lhs, const struct ceph_pg *rhs)
{
        if (lhs->pool < rhs->pool)
                return -1;
        if (lhs->pool > rhs->pool)
                return 1;
        if (lhs->seed < rhs->seed)
                return -1;
        if (lhs->seed > rhs->seed)
                return 1;

        return 0;
}

int ceph_spg_compare(const struct ceph_spg *lhs, const struct ceph_spg *rhs)
{
        int ret;

        ret = ceph_pg_compare(&lhs->pgid, &rhs->pgid);
        if (ret)
                return ret;

        if (lhs->shard < rhs->shard)
                return -1;
        if (lhs->shard > rhs->shard)
                return 1;

        return 0;
}

static struct ceph_pg_mapping *alloc_pg_mapping(size_t payload_len)
{
        struct ceph_pg_mapping *pg;

        pg = kmalloc(sizeof(*pg) + payload_len, GFP_NOIO);
        if (!pg)
                return NULL;

        RB_CLEAR_NODE(&pg->node);
        return pg;
}

static void free_pg_mapping(struct ceph_pg_mapping *pg)
{
        WARN_ON(!RB_EMPTY_NODE(&pg->node));

        kfree(pg);
}

/*
 * rbtree of pg_mapping for handling pg_temp (explicit mapping of pgid
 * to a set of osds) and primary_temp (explicit primary setting)
 */
DEFINE_RB_FUNCS2(pg_mapping, struct ceph_pg_mapping, pgid, ceph_pg_compare,
                 RB_BYPTR, const struct ceph_pg *, node)

/*
 * rbtree of pg pool info
 */
DEFINE_RB_FUNCS(pg_pool, struct ceph_pg_pool_info, id, node)

struct ceph_pg_pool_info *ceph_pg_pool_by_id(struct ceph_osdmap *map, u64 id)
{
        return lookup_pg_pool(&map->pg_pools, id);
}

const char *ceph_pg_pool_name_by_id(struct ceph_osdmap *map, u64 id)
{
        struct ceph_pg_pool_info *pi;

        if (id == CEPH_NOPOOL)
                return NULL;

        if (WARN_ON_ONCE(id > (u64) INT_MAX))
                return NULL;

        pi = lookup_pg_pool(&map->pg_pools, id);
        return pi ? pi->name : NULL;
}
EXPORT_SYMBOL(ceph_pg_pool_name_by_id);

int ceph_pg_poolid_by_name(struct ceph_osdmap *map, const char *name)
{
        struct rb_node *rbp;

        for (rbp = rb_first(&map->pg_pools); rbp; rbp = rb_next(rbp)) {
                struct ceph_pg_pool_info *pi =
                        rb_entry(rbp, struct ceph_pg_pool_info, node);
                if (pi->name && strcmp(pi->name, name) == 0)
                        return pi->id;
        }
        return -ENOENT;
}
EXPORT_SYMBOL(ceph_pg_poolid_by_name);

u64 ceph_pg_pool_flags(struct ceph_osdmap *map, u64 id)
{
        struct ceph_pg_pool_info *pi;

        pi = lookup_pg_pool(&map->pg_pools, id);
        return pi ? pi->flags : 0;
}
EXPORT_SYMBOL(ceph_pg_pool_flags);

static void __remove_pg_pool(struct rb_root *root, struct ceph_pg_pool_info *pi)
{
        erase_pg_pool(root, pi);
        kfree(pi->name);
        kfree(pi);
}

static int decode_pool(void **p, void *end, struct ceph_pg_pool_info *pi)
{
        u8 ev, cv;
        unsigned len, num;
        void *pool_end;

        ceph_decode_need(p, end, 2 + 4, bad);
        ev = ceph_decode_8(p);  /* encoding version */
        cv = ceph_decode_8(p); /* compat version */
        if (ev < 5) {
                pr_warn("got v %d < 5 cv %d of ceph_pg_pool\n", ev, cv);
                return -EINVAL;
        }
        if (cv > 9) {
                pr_warn("got v %d cv %d > 9 of ceph_pg_pool\n", ev, cv);
                return -EINVAL;
        }
        len = ceph_decode_32(p);
        ceph_decode_need(p, end, len, bad);
        pool_end = *p + len;

        pi->type = ceph_decode_8(p);
        pi->size = ceph_decode_8(p);
        pi->crush_ruleset = ceph_decode_8(p);
        pi->object_hash = ceph_decode_8(p);

        pi->pg_num = ceph_decode_32(p);
        pi->pgp_num = ceph_decode_32(p);

        *p += 4 + 4;  /* skip lpg* */
        *p += 4;      /* skip last_change */
        *p += 8 + 4;  /* skip snap_seq, snap_epoch */

        /* skip snaps */
        num = ceph_decode_32(p);
        while (num--) {
                *p += 8;  /* snapid key */
                *p += 1 + 1; /* versions */
                len = ceph_decode_32(p);
                *p += len;
        }

        /* skip removed_snaps */
        num = ceph_decode_32(p);
        *p += num * (8 + 8);

        *p += 8;  /* skip auid */
        pi->flags = ceph_decode_64(p);
        *p += 4;  /* skip crash_replay_interval */

        if (ev >= 7)
                pi->min_size = ceph_decode_8(p);
        else
                pi->min_size = pi->size - pi->size / 2;

        if (ev >= 8)
                *p += 8 + 8;  /* skip quota_max_* */

        if (ev >= 9) {
                /* skip tiers */
                num = ceph_decode_32(p);
                *p += num * 8;

                *p += 8;  /* skip tier_of */
                *p += 1;  /* skip cache_mode */

                pi->read_tier = ceph_decode_64(p);
                pi->write_tier = ceph_decode_64(p);
        } else {
                pi->read_tier = -1;
                pi->write_tier = -1;
        }

        if (ev >= 10) {
                /* skip properties */
                num = ceph_decode_32(p);
                while (num--) {
                        len = ceph_decode_32(p);
                        *p += len; /* key */
                        len = ceph_decode_32(p);
                        *p += len; /* val */
                }
        }

        if (ev >= 11) {
                /* skip hit_set_params */
                *p += 1 + 1; /* versions */
                len = ceph_decode_32(p);
                *p += len;

                *p += 4; /* skip hit_set_period */
                *p += 4; /* skip hit_set_count */
        }

        if (ev >= 12)
                *p += 4; /* skip stripe_width */

        if (ev >= 13) {
                *p += 8; /* skip target_max_bytes */
                *p += 8; /* skip target_max_objects */
                *p += 4; /* skip cache_target_dirty_ratio_micro */
                *p += 4; /* skip cache_target_full_ratio_micro */
                *p += 4; /* skip cache_min_flush_age */
                *p += 4; /* skip cache_min_evict_age */
        }

        if (ev >=  14) {
                /* skip erasure_code_profile */
                len = ceph_decode_32(p);
                *p += len;
        }

        /*
         * last_force_op_resend_preluminous, will be overridden if the
         * map was encoded with RESEND_ON_SPLIT
         */
        if (ev >= 15)
                pi->last_force_request_resend = ceph_decode_32(p);
        else
                pi->last_force_request_resend = 0;

        if (ev >= 16)
                *p += 4; /* skip min_read_recency_for_promote */

        if (ev >= 17)
                *p += 8; /* skip expected_num_objects */

        if (ev >= 19)
                *p += 4; /* skip cache_target_dirty_high_ratio_micro */

        if (ev >= 20)
                *p += 4; /* skip min_write_recency_for_promote */

        if (ev >= 21)
                *p += 1; /* skip use_gmt_hitset */

        if (ev >= 22)
                *p += 1; /* skip fast_read */

        if (ev >= 23) {
                *p += 4; /* skip hit_set_grade_decay_rate */
                *p += 4; /* skip hit_set_search_last_n */
        }

        if (ev >= 24) {
                /* skip opts */
                *p += 1 + 1; /* versions */
                len = ceph_decode_32(p);
                *p += len;
        }

        if (ev >= 25)
                pi->last_force_request_resend = ceph_decode_32(p);

        /* ignore the rest */

        *p = pool_end;
        calc_pg_masks(pi);
        return 0;

bad:
        return -EINVAL;
}

static int decode_pool_names(void **p, void *end, struct ceph_osdmap *map)
{
        struct ceph_pg_pool_info *pi;
        u32 num, len;
        u64 pool;

        ceph_decode_32_safe(p, end, num, bad);
        dout(" %d pool names\n", num);
        while (num--) {
                ceph_decode_64_safe(p, end, pool, bad);
                ceph_decode_32_safe(p, end, len, bad);
                dout("  pool %llu len %d\n", pool, len);
                ceph_decode_need(p, end, len, bad);
                pi = lookup_pg_pool(&map->pg_pools, pool);
                if (pi) {
                        char *name = kstrndup(*p, len, GFP_NOFS);

                        if (!name)
                                return -ENOMEM;
                        kfree(pi->name);
                        pi->name = name;
                        dout("  name is %s\n", pi->name);
                }
                *p += len;
        }
        return 0;

bad:
        return -EINVAL;
}

/*
 * CRUSH workspaces
 *
 * workspace_manager framework borrowed from fs/btrfs/compression.c.
 * Two simplifications: there is only one type of workspace and there
 * is always at least one workspace.
 */
static struct crush_work *alloc_workspace(const struct crush_map *c)
{
        struct crush_work *work;
        size_t work_size;

        WARN_ON(!c->working_size);
        work_size = crush_work_size(c, CEPH_PG_MAX_SIZE);
        dout("%s work_size %zu bytes\n", __func__, work_size);

        work = kvmalloc(work_size, GFP_NOIO);
        if (!work)
                return NULL;

        INIT_LIST_HEAD(&work->item);
        crush_init_workspace(c, work);
        return work;
}

static void free_workspace(struct crush_work *work)
{
        WARN_ON(!list_empty(&work->item));
        kvfree(work);
}

static void init_workspace_manager(struct workspace_manager *wsm)
{
        INIT_LIST_HEAD(&wsm->idle_ws);
        spin_lock_init(&wsm->ws_lock);
        atomic_set(&wsm->total_ws, 0);
        wsm->free_ws = 0;
        init_waitqueue_head(&wsm->ws_wait);
}

static void add_initial_workspace(struct workspace_manager *wsm,
                                  struct crush_work *work)
{
        WARN_ON(!list_empty(&wsm->idle_ws));

        list_add(&work->item, &wsm->idle_ws);
        atomic_set(&wsm->total_ws, 1);
        wsm->free_ws = 1;
}

static void cleanup_workspace_manager(struct workspace_manager *wsm)
{
        struct crush_work *work;

        while (!list_empty(&wsm->idle_ws)) {
                work = list_first_entry(&wsm->idle_ws, struct crush_work,
                                        item);
                list_del_init(&work->item);
                free_workspace(work);
        }
        atomic_set(&wsm->total_ws, 0);
        wsm->free_ws = 0;
}

/*
 * Finds an available workspace or allocates a new one.  If it's not
 * possible to allocate a new one, waits until there is one.
 */
static struct crush_work *get_workspace(struct workspace_manager *wsm,
                                        const struct crush_map *c)
{
        struct crush_work *work;
        int cpus = num_online_cpus();

again:
        spin_lock(&wsm->ws_lock);
        if (!list_empty(&wsm->idle_ws)) {
                work = list_first_entry(&wsm->idle_ws, struct crush_work,
                                        item);
                list_del_init(&work->item);
                wsm->free_ws--;
                spin_unlock(&wsm->ws_lock);
                return work;

        }
        if (atomic_read(&wsm->total_ws) > cpus) {
                DEFINE_WAIT(wait);

                spin_unlock(&wsm->ws_lock);
                prepare_to_wait(&wsm->ws_wait, &wait, TASK_UNINTERRUPTIBLE);
                if (atomic_read(&wsm->total_ws) > cpus && !wsm->free_ws)
                        schedule();
                finish_wait(&wsm->ws_wait, &wait);
                goto again;
        }
        atomic_inc(&wsm->total_ws);
        spin_unlock(&wsm->ws_lock);

        work = alloc_workspace(c);
        if (!work) {
                atomic_dec(&wsm->total_ws);
                wake_up(&wsm->ws_wait);

                /*
                 * Do not return the error but go back to waiting.  We
                 * have the initial workspace and the CRUSH computation
                 * time is bounded so we will get it eventually.
                 */
                WARN_ON(atomic_read(&wsm->total_ws) < 1);
                goto again;
        }
        return work;
}

/*
 * Puts a workspace back on the list or frees it if we have enough
 * idle ones sitting around.
 */
static void put_workspace(struct workspace_manager *wsm,
                          struct crush_work *work)
{
        spin_lock(&wsm->ws_lock);
        if (wsm->free_ws <= num_online_cpus()) {
                list_add(&work->item, &wsm->idle_ws);
                wsm->free_ws++;
                spin_unlock(&wsm->ws_lock);
                goto wake;
        }
        spin_unlock(&wsm->ws_lock);

        free_workspace(work);
        atomic_dec(&wsm->total_ws);
wake:
        if (wq_has_sleeper(&wsm->ws_wait))
                wake_up(&wsm->ws_wait);
}

/*
 * osd map
 */
struct ceph_osdmap *ceph_osdmap_alloc(void)
{
        struct ceph_osdmap *map;

        map = kzalloc(sizeof(*map), GFP_NOIO);
        if (!map)
                return NULL;

        map->pg_pools = RB_ROOT;
        map->pool_max = -1;
        map->pg_temp = RB_ROOT;
        map->primary_temp = RB_ROOT;
        map->pg_upmap = RB_ROOT;
        map->pg_upmap_items = RB_ROOT;

        init_workspace_manager(&map->crush_wsm);

        return map;
}

void ceph_osdmap_destroy(struct ceph_osdmap *map)
{
        dout("osdmap_destroy %p\n", map);

        if (map->crush)
                crush_destroy(map->crush);
        cleanup_workspace_manager(&map->crush_wsm);

        while (!RB_EMPTY_ROOT(&map->pg_temp)) {
                struct ceph_pg_mapping *pg =
                        rb_entry(rb_first(&map->pg_temp),
                                 struct ceph_pg_mapping, node);
                erase_pg_mapping(&map->pg_temp, pg);
                free_pg_mapping(pg);
        }
        while (!RB_EMPTY_ROOT(&map->primary_temp)) {
                struct ceph_pg_mapping *pg =
                        rb_entry(rb_first(&map->primary_temp),
                                 struct ceph_pg_mapping, node);
                erase_pg_mapping(&map->primary_temp, pg);
                free_pg_mapping(pg);
        }
        while (!RB_EMPTY_ROOT(&map->pg_upmap)) {
                struct ceph_pg_mapping *pg =
                        rb_entry(rb_first(&map->pg_upmap),
                                 struct ceph_pg_mapping, node);
                rb_erase(&pg->node, &map->pg_upmap);
                kfree(pg);
        }
        while (!RB_EMPTY_ROOT(&map->pg_upmap_items)) {
                struct ceph_pg_mapping *pg =
                        rb_entry(rb_first(&map->pg_upmap_items),
                                 struct ceph_pg_mapping, node);
                rb_erase(&pg->node, &map->pg_upmap_items);
                kfree(pg);
        }
        while (!RB_EMPTY_ROOT(&map->pg_pools)) {
                struct ceph_pg_pool_info *pi =
                        rb_entry(rb_first(&map->pg_pools),
                                 struct ceph_pg_pool_info, node);
                __remove_pg_pool(&map->pg_pools, pi);
        }
        kvfree(map->osd_state);
        kvfree(map->osd_weight);
        kvfree(map->osd_addr);
        kvfree(map->osd_primary_affinity);
        kfree(map);
}

/*
 * Adjust max_osd value, (re)allocate arrays.
 *
 * The new elements are properly initialized.
 */
static int osdmap_set_max_osd(struct ceph_osdmap *map, u32 max)
{
        u32 *state;
        u32 *weight;
        struct ceph_entity_addr *addr;
        u32 to_copy;
        int i;

        dout("%s old %u new %u\n", __func__, map->max_osd, max);
        if (max == map->max_osd)
                return 0;

        state = kvmalloc(array_size(max, sizeof(*state)), GFP_NOFS);
        weight = kvmalloc(array_size(max, sizeof(*weight)), GFP_NOFS);
        addr = kvmalloc(array_size(max, sizeof(*addr)), GFP_NOFS);
        if (!state || !weight || !addr) {
                kvfree(state);
                kvfree(weight);
                kvfree(addr);
                return -ENOMEM;
        }

        to_copy = min(map->max_osd, max);
        if (map->osd_state) {
                memcpy(state, map->osd_state, to_copy * sizeof(*state));
                memcpy(weight, map->osd_weight, to_copy * sizeof(*weight));
                memcpy(addr, map->osd_addr, to_copy * sizeof(*addr));
                kvfree(map->osd_state);
                kvfree(map->osd_weight);
                kvfree(map->osd_addr);
        }

        map->osd_state = state;
        map->osd_weight = weight;
        map->osd_addr = addr;
        for (i = map->max_osd; i < max; i++) {
                map->osd_state[i] = 0;
                map->osd_weight[i] = CEPH_OSD_OUT;
                memset(map->osd_addr + i, 0, sizeof(*map->osd_addr));
        }

        if (map->osd_primary_affinity) {
                u32 *affinity;

                affinity = kvmalloc(array_size(max, sizeof(*affinity)),
                                         GFP_NOFS);
                if (!affinity)
                        return -ENOMEM;

                memcpy(affinity, map->osd_primary_affinity,
                       to_copy * sizeof(*affinity));
                kvfree(map->osd_primary_affinity);

                map->osd_primary_affinity = affinity;
                for (i = map->max_osd; i < max; i++)
                        map->osd_primary_affinity[i] =
                            CEPH_OSD_DEFAULT_PRIMARY_AFFINITY;
        }

        map->max_osd = max;

        return 0;
}

static int osdmap_set_crush(struct ceph_osdmap *map, struct crush_map *crush)
{
        struct crush_work *work;

        if (IS_ERR(crush))
                return PTR_ERR(crush);

        work = alloc_workspace(crush);
        if (!work) {
                crush_destroy(crush);
                return -ENOMEM;
        }

        if (map->crush)
                crush_destroy(map->crush);
        cleanup_workspace_manager(&map->crush_wsm);
        map->crush = crush;
        add_initial_workspace(&map->crush_wsm, work);
        return 0;
}

#define OSDMAP_WRAPPER_COMPAT_VER        7
#define OSDMAP_CLIENT_DATA_COMPAT_VER        1

/*
 * Return 0 or error.  On success, *v is set to 0 for old (v6) osdmaps,
 * to struct_v of the client_data section for new (v7 and above)
 * osdmaps.
 */
static int get_osdmap_client_data_v(void **p, void *end,
                                    const char *prefix, u8 *v)
{
        u8 struct_v;

        ceph_decode_8_safe(p, end, struct_v, e_inval);
        if (struct_v >= 7) {
                u8 struct_compat;

                ceph_decode_8_safe(p, end, struct_compat, e_inval);
                if (struct_compat > OSDMAP_WRAPPER_COMPAT_VER) {
                        pr_warn("got v %d cv %d > %d of %s ceph_osdmap\n",
                                struct_v, struct_compat,
                                OSDMAP_WRAPPER_COMPAT_VER, prefix);
                        return -EINVAL;
                }
                *p += 4; /* ignore wrapper struct_len */

                ceph_decode_8_safe(p, end, struct_v, e_inval);
                ceph_decode_8_safe(p, end, struct_compat, e_inval);
                if (struct_compat > OSDMAP_CLIENT_DATA_COMPAT_VER) {
                        pr_warn("got v %d cv %d > %d of %s ceph_osdmap client data\n",
                                struct_v, struct_compat,
                                OSDMAP_CLIENT_DATA_COMPAT_VER, prefix);
                        return -EINVAL;
                }
                *p += 4; /* ignore client data struct_len */
        } else {
                u16 version;

                *p -= 1;
                ceph_decode_16_safe(p, end, version, e_inval);
                if (version < 6) {
                        pr_warn("got v %d < 6 of %s ceph_osdmap\n",
                                version, prefix);
                        return -EINVAL;
                }

                /* old osdmap encoding */
                struct_v = 0;
        }

        *v = struct_v;
        return 0;

e_inval:
        return -EINVAL;
}

static int __decode_pools(void **p, void *end, struct ceph_osdmap *map,
                          bool incremental)
{
        u32 n;

        ceph_decode_32_safe(p, end, n, e_inval);
        while (n--) {
                struct ceph_pg_pool_info *pi;
                u64 pool;
                int ret;

                ceph_decode_64_safe(p, end, pool, e_inval);

                pi = lookup_pg_pool(&map->pg_pools, pool);
                if (!incremental || !pi) {
                        pi = kzalloc(sizeof(*pi), GFP_NOFS);
                        if (!pi)
                                return -ENOMEM;

                        RB_CLEAR_NODE(&pi->node);
                        pi->id = pool;

                        if (!__insert_pg_pool(&map->pg_pools, pi)) {
                                kfree(pi);
                                return -EEXIST;
                        }
                }

                ret = decode_pool(p, end, pi);
                if (ret)
                        return ret;
        }

        return 0;

e_inval:
        return -EINVAL;
}

static int decode_pools(void **p, void *end, struct ceph_osdmap *map)
{
        return __decode_pools(p, end, map, false);
}

static int decode_new_pools(void **p, void *end, struct ceph_osdmap *map)
{
        return __decode_pools(p, end, map, true);
}

typedef struct ceph_pg_mapping *(*decode_mapping_fn_t)(void **, void *, bool);

static int decode_pg_mapping(void **p, void *end, struct rb_root *mapping_root,
                             decode_mapping_fn_t fn, bool incremental)
{
        u32 n;

        WARN_ON(!incremental && !fn);

        ceph_decode_32_safe(p, end, n, e_inval);
        while (n--) {
                struct ceph_pg_mapping *pg;
                struct ceph_pg pgid;
                int ret;

                ret = ceph_decode_pgid(p, end, &pgid);
                if (ret)
                        return ret;

                pg = lookup_pg_mapping(mapping_root, &pgid);
                if (pg) {
                        WARN_ON(!incremental);
                        erase_pg_mapping(mapping_root, pg);
                        free_pg_mapping(pg);
                }

                if (fn) {
                        pg = fn(p, end, incremental);
                        if (IS_ERR(pg))
                                return PTR_ERR(pg);

                        if (pg) {
                                pg->pgid = pgid; /* struct */
                                insert_pg_mapping(mapping_root, pg);
                        }
                }
        }

        return 0;

e_inval:
        return -EINVAL;
}

static struct ceph_pg_mapping *__decode_pg_temp(void **p, void *end,
                                                bool incremental)
{
        struct ceph_pg_mapping *pg;
        u32 len, i;

        ceph_decode_32_safe(p, end, len, e_inval);
        if (len == 0 && incremental)
                return NULL;        /* new_pg_temp: [] to remove */
        if (len > (SIZE_MAX - sizeof(*pg)) / sizeof(u32))
                return ERR_PTR(-EINVAL);

        ceph_decode_need(p, end, len * sizeof(u32), e_inval);
        pg = alloc_pg_mapping(len * sizeof(u32));
        if (!pg)
                return ERR_PTR(-ENOMEM);

        pg->pg_temp.len = len;
        for (i = 0; i < len; i++)
                pg->pg_temp.osds[i] = ceph_decode_32(p);

        return pg;

e_inval:
        return ERR_PTR(-EINVAL);
}

static int decode_pg_temp(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_temp, __decode_pg_temp,
                                 false);
}

static int decode_new_pg_temp(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_temp, __decode_pg_temp,
                                 true);
}

static struct ceph_pg_mapping *__decode_primary_temp(void **p, void *end,
                                                     bool incremental)
{
        struct ceph_pg_mapping *pg;
        u32 osd;

        ceph_decode_32_safe(p, end, osd, e_inval);
        if (osd == (u32)-1 && incremental)
                return NULL;        /* new_primary_temp: -1 to remove */

        pg = alloc_pg_mapping(0);
        if (!pg)
                return ERR_PTR(-ENOMEM);

        pg->primary_temp.osd = osd;
        return pg;

e_inval:
        return ERR_PTR(-EINVAL);
}

static int decode_primary_temp(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->primary_temp,
                                 __decode_primary_temp, false);
}

static int decode_new_primary_temp(void **p, void *end,
                                   struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->primary_temp,
                                 __decode_primary_temp, true);
}

u32 ceph_get_primary_affinity(struct ceph_osdmap *map, int osd)
{
        BUG_ON(osd >= map->max_osd);

        if (!map->osd_primary_affinity)
                return CEPH_OSD_DEFAULT_PRIMARY_AFFINITY;

        return map->osd_primary_affinity[osd];
}

static int set_primary_affinity(struct ceph_osdmap *map, int osd, u32 aff)
{
        BUG_ON(osd >= map->max_osd);

        if (!map->osd_primary_affinity) {
                int i;

                map->osd_primary_affinity = kvmalloc(
                    array_size(map->max_osd, sizeof(*map->osd_primary_affinity)),
                    GFP_NOFS);
                if (!map->osd_primary_affinity)
                        return -ENOMEM;

                for (i = 0; i < map->max_osd; i++)
                        map->osd_primary_affinity[i] =
                            CEPH_OSD_DEFAULT_PRIMARY_AFFINITY;
        }

        map->osd_primary_affinity[osd] = aff;

        return 0;
}

static int decode_primary_affinity(void **p, void *end,
                                   struct ceph_osdmap *map)
{
        u32 len, i;

        ceph_decode_32_safe(p, end, len, e_inval);
        if (len == 0) {
                kvfree(map->osd_primary_affinity);
                map->osd_primary_affinity = NULL;
                return 0;
        }
        if (len != map->max_osd)
                goto e_inval;

        ceph_decode_need(p, end, map->max_osd*sizeof(u32), e_inval);

        for (i = 0; i < map->max_osd; i++) {
                int ret;

                ret = set_primary_affinity(map, i, ceph_decode_32(p));
                if (ret)
                        return ret;
        }

        return 0;

e_inval:
        return -EINVAL;
}

static int decode_new_primary_affinity(void **p, void *end,
                                       struct ceph_osdmap *map)
{
        u32 n;

        ceph_decode_32_safe(p, end, n, e_inval);
        while (n--) {
                u32 osd, aff;
                int ret;

                ceph_decode_32_safe(p, end, osd, e_inval);
                ceph_decode_32_safe(p, end, aff, e_inval);

                ret = set_primary_affinity(map, osd, aff);
                if (ret)
                        return ret;

                osdmap_info(map, "osd%d primary-affinity 0x%x\n", osd, aff);
        }

        return 0;

e_inval:
        return -EINVAL;
}

static struct ceph_pg_mapping *__decode_pg_upmap(void **p, void *end,
                                                 bool __unused)
{
        return __decode_pg_temp(p, end, false);
}

static int decode_pg_upmap(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap, __decode_pg_upmap,
                                 false);
}

static int decode_new_pg_upmap(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap, __decode_pg_upmap,
                                 true);
}

static int decode_old_pg_upmap(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap, NULL, true);
}

static struct ceph_pg_mapping *__decode_pg_upmap_items(void **p, void *end,
                                                       bool __unused)
{
        struct ceph_pg_mapping *pg;
        u32 len, i;

        ceph_decode_32_safe(p, end, len, e_inval);
        if (len > (SIZE_MAX - sizeof(*pg)) / (2 * sizeof(u32)))
                return ERR_PTR(-EINVAL);

        ceph_decode_need(p, end, 2 * len * sizeof(u32), e_inval);
        pg = alloc_pg_mapping(2 * len * sizeof(u32));
        if (!pg)
                return ERR_PTR(-ENOMEM);

        pg->pg_upmap_items.len = len;
        for (i = 0; i < len; i++) {
                pg->pg_upmap_items.from_to[i][0] = ceph_decode_32(p);
                pg->pg_upmap_items.from_to[i][1] = ceph_decode_32(p);
        }

        return pg;

e_inval:
        return ERR_PTR(-EINVAL);
}

static int decode_pg_upmap_items(void **p, void *end, struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap_items,
                                 __decode_pg_upmap_items, false);
}

static int decode_new_pg_upmap_items(void **p, void *end,
                                     struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap_items,
                                 __decode_pg_upmap_items, true);
}

static int decode_old_pg_upmap_items(void **p, void *end,
                                     struct ceph_osdmap *map)
{
        return decode_pg_mapping(p, end, &map->pg_upmap_items, NULL, true);
}

/*
 * decode a full map.
 */
static int osdmap_decode(void **p, void *end, bool msgr2,
                         struct ceph_osdmap *map)
{
        u8 struct_v;
        u32 epoch = 0;
        void *start = *p;
        u32 max;
        u32 len, i;
        int err;

        dout("%s %p to %p len %d\n", __func__, *p, end, (int)(end - *p));

        err = get_osdmap_client_data_v(p, end, "full", &struct_v);
        if (err)
                goto bad;

        /* fsid, epoch, created, modified */
        ceph_decode_need(p, end, sizeof(map->fsid) + sizeof(u32) +
                         sizeof(map->created) + sizeof(map->modified), e_inval);
        ceph_decode_copy(p, &map->fsid, sizeof(map->fsid));
        epoch = map->epoch = ceph_decode_32(p);
        ceph_decode_copy(p, &map->created, sizeof(map->created));
        ceph_decode_copy(p, &map->modified, sizeof(map->modified));

        /* pools */
        err = decode_pools(p, end, map);
        if (err)
                goto bad;

        /* pool_name */
        err = decode_pool_names(p, end, map);
        if (err)
                goto bad;

        ceph_decode_32_safe(p, end, map->pool_max, e_inval);

        ceph_decode_32_safe(p, end, map->flags, e_inval);

        /* max_osd */
        ceph_decode_32_safe(p, end, max, e_inval);

        /* (re)alloc osd arrays */
        err = osdmap_set_max_osd(map, max);
        if (err)
                goto bad;

        /* osd_state, osd_weight, osd_addrs->client_addr */
        ceph_decode_need(p, end, 3*sizeof(u32) +
                         map->max_osd*(struct_v >= 5 ? sizeof(u32) :
                                                       sizeof(u8)) +
                                       sizeof(*map->osd_weight), e_inval);
        if (ceph_decode_32(p) != map->max_osd)
                goto e_inval;

        if (struct_v >= 5) {
                for (i = 0; i < map->max_osd; i++)
                        map->osd_state[i] = ceph_decode_32(p);
        } else {
                for (i = 0; i < map->max_osd; i++)
                        map->osd_state[i] = ceph_decode_8(p);
        }

        if (ceph_decode_32(p) != map->max_osd)
                goto e_inval;

        for (i = 0; i < map->max_osd; i++)
                map->osd_weight[i] = ceph_decode_32(p);

        if (ceph_decode_32(p) != map->max_osd)
                goto e_inval;

        for (i = 0; i < map->max_osd; i++) {
                struct ceph_entity_addr *addr = &map->osd_addr[i];

                if (struct_v >= 8)
                        err = ceph_decode_entity_addrvec(p, end, msgr2, addr);
                else
                        err = ceph_decode_entity_addr(p, end, addr);
                if (err)
                        goto bad;

                dout("%s osd%d addr %s\n", __func__, i, ceph_pr_addr(addr));
        }

        /* pg_temp */
        err = decode_pg_temp(p, end, map);
        if (err)
                goto bad;

        /* primary_temp */
        if (struct_v >= 1) {
                err = decode_primary_temp(p, end, map);
                if (err)
                        goto bad;
        }

        /* primary_affinity */
        if (struct_v >= 2) {
                err = decode_primary_affinity(p, end, map);
                if (err)
                        goto bad;
        } else {
                WARN_ON(map->osd_primary_affinity);
        }

        /* crush */
        ceph_decode_32_safe(p, end, len, e_inval);
        err = osdmap_set_crush(map, crush_decode(*p, min(*p + len, end)));
        if (err)
                goto bad;

        *p += len;
        if (struct_v >= 3) {
                /* erasure_code_profiles */
                ceph_decode_skip_map_of_map(p, end, string, string, string,
                                            e_inval);
        }

        if (struct_v >= 4) {
                err = decode_pg_upmap(p, end, map);
                if (err)
                        goto bad;

                err = decode_pg_upmap_items(p, end, map);
                if (err)
                        goto bad;
        } else {
                WARN_ON(!RB_EMPTY_ROOT(&map->pg_upmap));
                WARN_ON(!RB_EMPTY_ROOT(&map->pg_upmap_items));
        }

        /* ignore the rest */
        *p = end;

        dout("full osdmap epoch %d max_osd %d\n", map->epoch, map->max_osd);
        return 0;

e_inval:
        err = -EINVAL;
bad:
        pr_err("corrupt full osdmap (%d) epoch %d off %d (%p of %p-%p)\n",
               err, epoch, (int)(*p - start), *p, start, end);
        print_hex_dump(KERN_DEBUG, "osdmap: ",
                       DUMP_PREFIX_OFFSET, 16, 1,
                       start, end - start, true);
        return err;
}

/*
 * Allocate and decode a full map.
 */
struct ceph_osdmap *ceph_osdmap_decode(void **p, void *end, bool msgr2)
{
        struct ceph_osdmap *map;
        int ret;

        map = ceph_osdmap_alloc();
        if (!map)
                return ERR_PTR(-ENOMEM);

        ret = osdmap_decode(p, end, msgr2, map);
        if (ret) {
                ceph_osdmap_destroy(map);
                return ERR_PTR(ret);
        }

        return map;
}

/*
 * Encoding order is (new_up_client, new_state, new_weight).  Need to
 * apply in the (new_weight, new_state, new_up_client) order, because
 * an incremental map may look like e.g.
 *
 *     new_up_client: { osd=6, addr=... } # set osd_state and addr
 *     new_state: { osd=6, xorstate=EXISTS } # clear osd_state
 */
static int decode_new_up_state_weight(void **p, void *end, u8 struct_v,
                                      bool msgr2, struct ceph_osdmap *map)
{
        void *new_up_client;
        void *new_state;
        void *new_weight_end;
        u32 len;
        int ret;
        int i;

        new_up_client = *p;
        ceph_decode_32_safe(p, end, len, e_inval);
        for (i = 0; i < len; ++i) {
                struct ceph_entity_addr addr;

                ceph_decode_skip_32(p, end, e_inval);
                if (struct_v >= 7)
                        ret = ceph_decode_entity_addrvec(p, end, msgr2, &addr);
                else
                        ret = ceph_decode_entity_addr(p, end, &addr);
                if (ret)
                        return ret;
        }

        new_state = *p;
        ceph_decode_32_safe(p, end, len, e_inval);
        len *= sizeof(u32) + (struct_v >= 5 ? sizeof(u32) : sizeof(u8));
        ceph_decode_need(p, end, len, e_inval);
        *p += len;

        /* new_weight */
        ceph_decode_32_safe(p, end, len, e_inval);
        while (len--) {
                s32 osd;
                u32 w;

                ceph_decode_need(p, end, 2*sizeof(u32), e_inval);
                osd = ceph_decode_32(p);
                w = ceph_decode_32(p);
                BUG_ON(osd >= map->max_osd);
                osdmap_info(map, "osd%d weight 0x%x %s\n", osd, w,
                            w == CEPH_OSD_IN ? "(in)" :
                            (w == CEPH_OSD_OUT ? "(out)" : ""));
                map->osd_weight[osd] = w;

                /*
                 * If we are marking in, set the EXISTS, and clear the
                 * AUTOOUT and NEW bits.
                 */
                if (w) {
                        map->osd_state[osd] |= CEPH_OSD_EXISTS;
                        map->osd_state[osd] &= ~(CEPH_OSD_AUTOOUT |
                                                 CEPH_OSD_NEW);
                }
        }
        new_weight_end = *p;

        /* new_state (up/down) */
        *p = new_state;
        len = ceph_decode_32(p);
        while (len--) {
                s32 osd;
                u32 xorstate;

                osd = ceph_decode_32(p);
                if (struct_v >= 5)
                        xorstate = ceph_decode_32(p);
                else
                        xorstate = ceph_decode_8(p);
                if (xorstate == 0)
                        xorstate = CEPH_OSD_UP;
                BUG_ON(osd >= map->max_osd);
                if ((map->osd_state[osd] & CEPH_OSD_UP) &&
                    (xorstate & CEPH_OSD_UP))
                        osdmap_info(map, "osd%d down\n", osd);
                if ((map->osd_state[osd] & CEPH_OSD_EXISTS) &&
                    (xorstate & CEPH_OSD_EXISTS)) {
                        osdmap_info(map, "osd%d does not exist\n", osd);
                        ret = set_primary_affinity(map, osd,
                                                   CEPH_OSD_DEFAULT_PRIMARY_AFFINITY);
                        if (ret)
                                return ret;
                        memset(map->osd_addr + osd, 0, sizeof(*map->osd_addr));
                        map->osd_state[osd] = 0;
                } else {
                        map->osd_state[osd] ^= xorstate;
                }
        }

        /* new_up_client */
        *p = new_up_client;
        len = ceph_decode_32(p);
        while (len--) {
                s32 osd;
                struct ceph_entity_addr addr;

                osd = ceph_decode_32(p);
                BUG_ON(osd >= map->max_osd);
                if (struct_v >= 7)
                        ret = ceph_decode_entity_addrvec(p, end, msgr2, &addr);
                else
                        ret = ceph_decode_entity_addr(p, end, &addr);
                if (ret)
                        return ret;

                dout("%s osd%d addr %s\n", __func__, osd, ceph_pr_addr(&addr));

                osdmap_info(map, "osd%d up\n", osd);
                map->osd_state[osd] |= CEPH_OSD_EXISTS | CEPH_OSD_UP;
                map->osd_addr[osd] = addr;
        }

        *p = new_weight_end;
        return 0;

e_inval:
        return -EINVAL;
}

/*
 * decode and apply an incremental map update.
 */
struct ceph_osdmap *osdmap_apply_incremental(void **p, void *end, bool msgr2,
                                             struct ceph_osdmap *map)
{
        struct ceph_fsid fsid;
        u32 epoch = 0;
        struct ceph_timespec modified;
        s32 len;
        u64 pool;
        __s64 new_pool_max;
        __s32 new_flags, max;
        void *start = *p;
        int err;
        u8 struct_v;

        dout("%s %p to %p len %d\n", __func__, *p, end, (int)(end - *p));

        err = get_osdmap_client_data_v(p, end, "inc", &struct_v);
        if (err)
                goto bad;

        /* fsid, epoch, modified, new_pool_max, new_flags */
        ceph_decode_need(p, end, sizeof(fsid) + sizeof(u32) + sizeof(modified) +
                         sizeof(u64) + sizeof(u32), e_inval);
        ceph_decode_copy(p, &fsid, sizeof(fsid));
        epoch = ceph_decode_32(p);
        BUG_ON(epoch != map->epoch+1);
        ceph_decode_copy(p, &modified, sizeof(modified));
        new_pool_max = ceph_decode_64(p);
        new_flags = ceph_decode_32(p);

        /* full map? */
        ceph_decode_32_safe(p, end, len, e_inval);
        if (len > 0) {
                dout("apply_incremental full map len %d, %p to %p\n",
                     len, *p, end);
                return ceph_osdmap_decode(p, min(*p+len, end), msgr2);
        }

        /* new crush? */
        ceph_decode_32_safe(p, end, len, e_inval);
        if (len > 0) {
                err = osdmap_set_crush(map,
                                       crush_decode(*p, min(*p + len, end)));
                if (err)
                        goto bad;
                *p += len;
        }

        /* new flags? */
        if (new_flags >= 0)
                map->flags = new_flags;
        if (new_pool_max >= 0)
                map->pool_max = new_pool_max;

        /* new max? */
        ceph_decode_32_safe(p, end, max, e_inval);
        if (max >= 0) {
                err = osdmap_set_max_osd(map, max);
                if (err)
                        goto bad;
        }

        map->epoch++;
        map->modified = modified;

        /* new_pools */
        err = decode_new_pools(p, end, map);
        if (err)
                goto bad;

        /* new_pool_names */
        err = decode_pool_names(p, end, map);
        if (err)
                goto bad;

        /* old_pool */
        ceph_decode_32_safe(p, end, len, e_inval);
        while (len--) {
                struct ceph_pg_pool_info *pi;

                ceph_decode_64_safe(p, end, pool, e_inval);
                pi = lookup_pg_pool(&map->pg_pools, pool);
                if (pi)
                        __remove_pg_pool(&map->pg_pools, pi);
        }

        /* new_up_client, new_state, new_weight */
        err = decode_new_up_state_weight(p, end, struct_v, msgr2, map);
        if (err)
                goto bad;

        /* new_pg_temp */
        err = decode_new_pg_temp(p, end, map);
        if (err)
                goto bad;

        /* new_primary_temp */
        if (struct_v >= 1) {
                err = decode_new_primary_temp(p, end, map);
                if (err)
                        goto bad;
        }

        /* new_primary_affinity */
        if (struct_v >= 2) {
                err = decode_new_primary_affinity(p, end, map);
                if (err)
                        goto bad;
        }

        if (struct_v >= 3) {
                /* new_erasure_code_profiles */
                ceph_decode_skip_map_of_map(p, end, string, string, string,
                                            e_inval);
                /* old_erasure_code_profiles */
                ceph_decode_skip_set(p, end, string, e_inval);
        }

        if (struct_v >= 4) {
                err = decode_new_pg_upmap(p, end, map);
                if (err)
                        goto bad;

                err = decode_old_pg_upmap(p, end, map);
                if (err)
                        goto bad;

                err = decode_new_pg_upmap_items(p, end, map);
                if (err)
                        goto bad;

                err = decode_old_pg_upmap_items(p, end, map);
                if (err)
                        goto bad;
        }

        /* ignore the rest */
        *p = end;

        dout("inc osdmap epoch %d max_osd %d\n", map->epoch, map->max_osd);
        return map;

e_inval:
        err = -EINVAL;
bad:
        pr_err("corrupt inc osdmap (%d) epoch %d off %d (%p of %p-%p)\n",
               err, epoch, (int)(*p - start), *p, start, end);
        print_hex_dump(KERN_DEBUG, "osdmap: ",
                       DUMP_PREFIX_OFFSET, 16, 1,
                       start, end - start, true);
        return ERR_PTR(err);
}

void ceph_oloc_copy(struct ceph_object_locator *dest,
                    const struct ceph_object_locator *src)
{
        ceph_oloc_destroy(dest);

        dest->pool = src->pool;
        if (src->pool_ns)
                dest->pool_ns = ceph_get_string(src->pool_ns);
        else
                dest->pool_ns = NULL;
}
EXPORT_SYMBOL(ceph_oloc_copy);

void ceph_oloc_destroy(struct ceph_object_locator *oloc)
{
        ceph_put_string(oloc->pool_ns);
}
EXPORT_SYMBOL(ceph_oloc_destroy);

void ceph_oid_copy(struct ceph_object_id *dest,
                   const struct ceph_object_id *src)
{
        ceph_oid_destroy(dest);

        if (src->name != src->inline_name) {
                /* very rare, see ceph_object_id definition */
                dest->name = kmalloc(src->name_len + 1,
                                     GFP_NOIO | __GFP_NOFAIL);
        } else {
                dest->name = dest->inline_name;
        }
        memcpy(dest->name, src->name, src->name_len + 1);
        dest->name_len = src->name_len;
}
EXPORT_SYMBOL(ceph_oid_copy);

static __printf(2, 0)
int oid_printf_vargs(struct ceph_object_id *oid, const char *fmt, va_list ap)
{
        int len;

        WARN_ON(!ceph_oid_empty(oid));

        len = vsnprintf(oid->inline_name, sizeof(oid->inline_name), fmt, ap);
        if (len >= sizeof(oid->inline_name))
                return len;

        oid->name_len = len;
        return 0;
}

/*
 * If oid doesn't fit into inline buffer, BUG.
 */
void ceph_oid_printf(struct ceph_object_id *oid, const char *fmt, ...)
{
        va_list ap;

        va_start(ap, fmt);
        BUG_ON(oid_printf_vargs(oid, fmt, ap));
        va_end(ap);
}
EXPORT_SYMBOL(ceph_oid_printf);

static __printf(3, 0)
int oid_aprintf_vargs(struct ceph_object_id *oid, gfp_t gfp,
                      const char *fmt, va_list ap)
{
        va_list aq;
        int len;

        va_copy(aq, ap);
        len = oid_printf_vargs(oid, fmt, aq);
        va_end(aq);

        if (len) {
                char *external_name;

                external_name = kmalloc(len + 1, gfp);
                if (!external_name)
                        return -ENOMEM;

                oid->name = external_name;
                WARN_ON(vsnprintf(oid->name, len + 1, fmt, ap) != len);
                oid->name_len = len;
        }

        return 0;
}

/*
 * If oid doesn't fit into inline buffer, allocate.
 */
int ceph_oid_aprintf(struct ceph_object_id *oid, gfp_t gfp,
                     const char *fmt, ...)
{
        va_list ap;
        int ret;

        va_start(ap, fmt);
        ret = oid_aprintf_vargs(oid, gfp, fmt, ap);
        va_end(ap);

        return ret;
}
EXPORT_SYMBOL(ceph_oid_aprintf);

void ceph_oid_destroy(struct ceph_object_id *oid)
{
        if (oid->name != oid->inline_name)
                kfree(oid->name);
}
EXPORT_SYMBOL(ceph_oid_destroy);

/*
 * osds only
 */
static bool __osds_equal(const struct ceph_osds *lhs,
                         const struct ceph_osds *rhs)
{
        if (lhs->size == rhs->size &&
            !memcmp(lhs->osds, rhs->osds, rhs->size * sizeof(rhs->osds[0])))
                return true;

        return false;
}

/*
 * osds + primary
 */
static bool osds_equal(const struct ceph_osds *lhs,
                       const struct ceph_osds *rhs)
{
        if (__osds_equal(lhs, rhs) &&
            lhs->primary == rhs->primary)
                return true;

        return false;
}

static bool osds_valid(const struct ceph_osds *set)
{
        /* non-empty set */
        if (set->size > 0 && set->primary >= 0)
                return true;

        /* empty can_shift_osds set */
        if (!set->size && set->primary == -1)
                return true;

        /* empty !can_shift_osds set - all NONE */
        if (set->size > 0 && set->primary == -1) {
                int i;

                for (i = 0; i < set->size; i++) {
                        if (set->osds[i] != CRUSH_ITEM_NONE)
                                break;
                }
                if (i == set->size)
                        return true;
        }

        return false;
}

void ceph_osds_copy(struct ceph_osds *dest, const struct ceph_osds *src)
{
        memcpy(dest->osds, src->osds, src->size * sizeof(src->osds[0]));
        dest->size = src->size;
        dest->primary = src->primary;
}

bool ceph_pg_is_split(const struct ceph_pg *pgid, u32 old_pg_num,
                      u32 new_pg_num)
{
        int old_bits = calc_bits_of(old_pg_num);
        int old_mask = (1 << old_bits) - 1;
        int n;

        WARN_ON(pgid->seed >= old_pg_num);
        if (new_pg_num <= old_pg_num)
                return false;

        for (n = 1; ; n++) {
                int next_bit = n << (old_bits - 1);
                u32 s = next_bit | pgid->seed;

                if (s < old_pg_num || s == pgid->seed)
                        continue;
                if (s >= new_pg_num)
                        break;

                s = ceph_stable_mod(s, old_pg_num, old_mask);
                if (s == pgid->seed)
                        return true;
        }

        return false;
}

bool ceph_is_new_interval(const struct ceph_osds *old_acting,
                          const struct ceph_osds *new_acting,
                          const struct ceph_osds *old_up,
                          const struct ceph_osds *new_up,
                          int old_size,
                          int new_size,
                          int old_min_size,
                          int new_min_size,
                          u32 old_pg_num,
                          u32 new_pg_num,
                          bool old_sort_bitwise,
                          bool new_sort_bitwise,
                          bool old_recovery_deletes,
                          bool new_recovery_deletes,
                          const struct ceph_pg *pgid)
{
        return !osds_equal(old_acting, new_acting) ||
               !osds_equal(old_up, new_up) ||
               old_size != new_size ||
               old_min_size != new_min_size ||
               ceph_pg_is_split(pgid, old_pg_num, new_pg_num) ||
               old_sort_bitwise != new_sort_bitwise ||
               old_recovery_deletes != new_recovery_deletes;
}

static int calc_pg_rank(int osd, const struct ceph_osds *acting)
{
        int i;

        for (i = 0; i < acting->size; i++) {
                if (acting->osds[i] == osd)
                        return i;
        }

        return -1;
}

static bool primary_changed(const struct ceph_osds *old_acting,
                            const struct ceph_osds *new_acting)
{
        if (!old_acting->size && !new_acting->size)
                return false; /* both still empty */

        if (!old_acting->size ^ !new_acting->size)
                return true; /* was empty, now not, or vice versa */

        if (old_acting->primary != new_acting->primary)
                return true; /* primary changed */

        if (calc_pg_rank(old_acting->primary, old_acting) !=
            calc_pg_rank(new_acting->primary, new_acting))
                return true;

        return false; /* same primary (tho replicas may have changed) */
}

bool ceph_osds_changed(const struct ceph_osds *old_acting,
                       const struct ceph_osds *new_acting,
                       bool any_change)
{
        if (primary_changed(old_acting, new_acting))
                return true;

        if (any_change && !__osds_equal(old_acting, new_acting))
                return true;

        return false;
}

/*
 * Map an object into a PG.
 *
 * Should only be called with target_oid and target_oloc (as opposed to
 * base_oid and base_oloc), since tiering isn't taken into account.
 */
void __ceph_object_locator_to_pg(struct ceph_pg_pool_info *pi,
                                 const struct ceph_object_id *oid,
                                 const struct ceph_object_locator *oloc,
                                 struct ceph_pg *raw_pgid)
{
        WARN_ON(pi->id != oloc->pool);

        if (!oloc->pool_ns) {
                raw_pgid->pool = oloc->pool;
                raw_pgid->seed = ceph_str_hash(pi->object_hash, oid->name,
                                             oid->name_len);
                dout("%s %s -> raw_pgid %llu.%x\n", __func__, oid->name,
                     raw_pgid->pool, raw_pgid->seed);
        } else {
                char stack_buf[256];
                char *buf = stack_buf;
                int nsl = oloc->pool_ns->len;
                size_t total = nsl + 1 + oid->name_len;

                if (total > sizeof(stack_buf))
                        buf = kmalloc(total, GFP_NOIO | __GFP_NOFAIL);
                memcpy(buf, oloc->pool_ns->str, nsl);
                buf[nsl] = '\037';
                memcpy(buf + nsl + 1, oid->name, oid->name_len);
                raw_pgid->pool = oloc->pool;
                raw_pgid->seed = ceph_str_hash(pi->object_hash, buf, total);
                if (buf != stack_buf)
                        kfree(buf);
                dout("%s %s ns %.*s -> raw_pgid %llu.%x\n", __func__,
                     oid->name, nsl, oloc->pool_ns->str,
                     raw_pgid->pool, raw_pgid->seed);
        }
}

int ceph_object_locator_to_pg(struct ceph_osdmap *osdmap,
                              const struct ceph_object_id *oid,
                              const struct ceph_object_locator *oloc,
                              struct ceph_pg *raw_pgid)
{
        struct ceph_pg_pool_info *pi;

        pi = ceph_pg_pool_by_id(osdmap, oloc->pool);
        if (!pi)
                return -ENOENT;

        __ceph_object_locator_to_pg(pi, oid, oloc, raw_pgid);
        return 0;
}
EXPORT_SYMBOL(ceph_object_locator_to_pg);

/*
 * Map a raw PG (full precision ps) into an actual PG.
 */
static void raw_pg_to_pg(struct ceph_pg_pool_info *pi,
                         const struct ceph_pg *raw_pgid,
                         struct ceph_pg *pgid)
{
        pgid->pool = raw_pgid->pool;
        pgid->seed = ceph_stable_mod(raw_pgid->seed, pi->pg_num,
                                     pi->pg_num_mask);
}

/*
 * Map a raw PG (full precision ps) into a placement ps (placement
 * seed).  Include pool id in that value so that different pools don't
 * use the same seeds.
 */
static u32 raw_pg_to_pps(struct ceph_pg_pool_info *pi,
                         const struct ceph_pg *raw_pgid)
{
        if (pi->flags & CEPH_POOL_FLAG_HASHPSPOOL) {
                /* hash pool id and seed so that pool PGs do not overlap */
                return crush_hash32_2(CRUSH_HASH_RJENKINS1,
                                      ceph_stable_mod(raw_pgid->seed,
                                                      pi->pgp_num,
                                                      pi->pgp_num_mask),
                                      raw_pgid->pool);
        } else {
                /*
                 * legacy behavior: add ps and pool together.  this is
                 * not a great approach because the PGs from each pool
                 * will overlap on top of each other: 0.5 == 1.4 ==
                 * 2.3 == ...
                 */
                return ceph_stable_mod(raw_pgid->seed, pi->pgp_num,
                                       pi->pgp_num_mask) +
                       (unsigned)raw_pgid->pool;
        }
}

/*
 * Magic value used for a "default" fallback choose_args, used if the
 * crush_choose_arg_map passed to do_crush() does not exist.  If this
 * also doesn't exist, fall back to canonical weights.
 */
#define CEPH_DEFAULT_CHOOSE_ARGS        -1

static int do_crush(struct ceph_osdmap *map, int ruleno, int x,
                    int *result, int result_max,
                    const __u32 *weight, int weight_max,
                    s64 choose_args_index)
{
        struct crush_choose_arg_map *arg_map;
        struct crush_work *work;
        int r;

        BUG_ON(result_max > CEPH_PG_MAX_SIZE);

        arg_map = lookup_choose_arg_map(&map->crush->choose_args,
                                        choose_args_index);
        if (!arg_map)
                arg_map = lookup_choose_arg_map(&map->crush->choose_args,
                                                CEPH_DEFAULT_CHOOSE_ARGS);

        work = get_workspace(&map->crush_wsm, map->crush);
        r = crush_do_rule(map->crush, ruleno, x, result, result_max,
                          weight, weight_max, work,
                          arg_map ? arg_map->args : NULL);
        put_workspace(&map->crush_wsm, work);
        return r;
}

static void remove_nonexistent_osds(struct ceph_osdmap *osdmap,
                                    struct ceph_pg_pool_info *pi,
                                    struct ceph_osds *set)
{
        int i;

        if (ceph_can_shift_osds(pi)) {
                int removed = 0;

                /* shift left */
                for (i = 0; i < set->size; i++) {
                        if (!ceph_osd_exists(osdmap, set->osds[i])) {
                                removed++;
                                continue;
                        }
                        if (removed)
                                set->osds[i - removed] = set->osds[i];
                }
                set->size -= removed;
        } else {
                /* set dne devices to NONE */
                for (i = 0; i < set->size; i++) {
                        if (!ceph_osd_exists(osdmap, set->osds[i]))
                                set->osds[i] = CRUSH_ITEM_NONE;
                }
        }
}

/*
 * Calculate raw set (CRUSH output) for given PG and filter out
 * nonexistent OSDs.  ->primary is undefined for a raw set.
 *
 * Placement seed (CRUSH input) is returned through @ppps.
 */
static void pg_to_raw_osds(struct ceph_osdmap *osdmap,
                           struct ceph_pg_pool_info *pi,
                           const struct ceph_pg *raw_pgid,
                           struct ceph_osds *raw,
                           u32 *ppps)
{
        u32 pps = raw_pg_to_pps(pi, raw_pgid);
        int ruleno;
        int len;

        ceph_osds_init(raw);
        if (ppps)
                *ppps = pps;

        ruleno = crush_find_rule(osdmap->crush, pi->crush_ruleset, pi->type,
                                 pi->size);
        if (ruleno < 0) {
                pr_err("no crush rule: pool %lld ruleset %d type %d size %d\n",
                       pi->id, pi->crush_ruleset, pi->type, pi->size);
                return;
        }

        if (pi->size > ARRAY_SIZE(raw->osds)) {
                pr_err_ratelimited("pool %lld ruleset %d type %d too wide: size %d > %zu\n",
                       pi->id, pi->crush_ruleset, pi->type, pi->size,
                       ARRAY_SIZE(raw->osds));
                return;
        }

        len = do_crush(osdmap, ruleno, pps, raw->osds, pi->size,
                       osdmap->osd_weight, osdmap->max_osd, pi->id);
        if (len < 0) {
                pr_err("error %d from crush rule %d: pool %lld ruleset %d type %d size %d\n",
                       len, ruleno, pi->id, pi->crush_ruleset, pi->type,
                       pi->size);
                return;
        }

        raw->size = len;
        remove_nonexistent_osds(osdmap, pi, raw);
}

/* apply pg_upmap[_items] mappings */
static void apply_upmap(struct ceph_osdmap *osdmap,
                        const struct ceph_pg *pgid,
                        struct ceph_osds *raw)
{
        struct ceph_pg_mapping *pg;
        int i, j;

        pg = lookup_pg_mapping(&osdmap->pg_upmap, pgid);
        if (pg) {
                /* make sure targets aren't marked out */
                for (i = 0; i < pg->pg_upmap.len; i++) {
                        int osd = pg->pg_upmap.osds[i];

                        if (osd != CRUSH_ITEM_NONE &&
                            osd < osdmap->max_osd &&
                            osdmap->osd_weight[osd] == 0) {
                                /* reject/ignore explicit mapping */
                                return;
                        }
                }
                for (i = 0; i < pg->pg_upmap.len; i++)
                        raw->osds[i] = pg->pg_upmap.osds[i];
                raw->size = pg->pg_upmap.len;
                /* check and apply pg_upmap_items, if any */
        }

        pg = lookup_pg_mapping(&osdmap->pg_upmap_items, pgid);
        if (pg) {
                /*
                 * Note: this approach does not allow a bidirectional swap,
                 * e.g., [[1,2],[2,1]] applied to [0,1,2] -> [0,2,1].
                 */
                for (i = 0; i < pg->pg_upmap_items.len; i++) {
                        int from = pg->pg_upmap_items.from_to[i][0];
                        int to = pg->pg_upmap_items.from_to[i][1];
                        int pos = -1;
                        bool exists = false;

                        /* make sure replacement doesn't already appear */
                        for (j = 0; j < raw->size; j++) {
                                int osd = raw->osds[j];

                                if (osd == to) {
                                        exists = true;
                                        break;
                                }
                                /* ignore mapping if target is marked out */
                                if (osd == from && pos < 0 &&
                                    !(to != CRUSH_ITEM_NONE &&
                                      to < osdmap->max_osd &&
                                      osdmap->osd_weight[to] == 0)) {
                                        pos = j;
                                }
                        }
                        if (!exists && pos >= 0)
                                raw->osds[pos] = to;
                }
        }
}

/*
 * Given raw set, calculate up set and up primary.  By definition of an
 * up set, the result won't contain nonexistent or down OSDs.
 *
 * This is done in-place - on return @set is the up set.  If it's
 * empty, ->primary will remain undefined.
 */
static void raw_to_up_osds(struct ceph_osdmap *osdmap,
                           struct ceph_pg_pool_info *pi,
                           struct ceph_osds *set)
{
        int i;

        /* ->primary is undefined for a raw set */
        BUG_ON(set->primary != -1);

        if (ceph_can_shift_osds(pi)) {
                int removed = 0;

                /* shift left */
                for (i = 0; i < set->size; i++) {
                        if (ceph_osd_is_down(osdmap, set->osds[i])) {
                                removed++;
                                continue;
                        }
                        if (removed)
                                set->osds[i - removed] = set->osds[i];
                }
                set->size -= removed;
                if (set->size > 0)
                        set->primary = set->osds[0];
        } else {
                /* set down/dne devices to NONE */
                for (i = set->size - 1; i >= 0; i--) {
                        if (ceph_osd_is_down(osdmap, set->osds[i]))
                                set->osds[i] = CRUSH_ITEM_NONE;
                        else
                                set->primary = set->osds[i];
                }
        }
}

static void apply_primary_affinity(struct ceph_osdmap *osdmap,
                                   struct ceph_pg_pool_info *pi,
                                   u32 pps,
                                   struct ceph_osds *up)
{
        int i;
        int pos = -1;

        /*
         * Do we have any non-default primary_affinity values for these
         * osds?
         */
        if (!osdmap->osd_primary_affinity)
                return;

        for (i = 0; i < up->size; i++) {
                int osd = up->osds[i];

                if (osd != CRUSH_ITEM_NONE &&
                    osdmap->osd_primary_affinity[osd] !=
                                        CEPH_OSD_DEFAULT_PRIMARY_AFFINITY) {
                        break;
                }
        }
        if (i == up->size)
                return;

        /*
         * Pick the primary.  Feed both the seed (for the pg) and the
         * osd into the hash/rng so that a proportional fraction of an
         * osd's pgs get rejected as primary.
         */
        for (i = 0; i < up->size; i++) {
                int osd = up->osds[i];
                u32 aff;

                if (osd == CRUSH_ITEM_NONE)
                        continue;

                aff = osdmap->osd_primary_affinity[osd];
                if (aff < CEPH_OSD_MAX_PRIMARY_AFFINITY &&
                    (crush_hash32_2(CRUSH_HASH_RJENKINS1,
                                    pps, osd) >> 16) >= aff) {
                        /*
                         * We chose not to use this primary.  Note it
                         * anyway as a fallback in case we don't pick
                         * anyone else, but keep looking.
                         */
                        if (pos < 0)
                                pos = i;
                } else {
                        pos = i;
                        break;
                }
        }
        if (pos < 0)
                return;

        up->primary = up->osds[pos];

        if (ceph_can_shift_osds(pi) && pos > 0) {
                /* move the new primary to the front */
                for (i = pos; i > 0; i--)
                        up->osds[i] = up->osds[i - 1];
                up->osds[0] = up->primary;
        }
}

/*
 * Get pg_temp and primary_temp mappings for given PG.
 *
 * Note that a PG may have none, only pg_temp, only primary_temp or
 * both pg_temp and primary_temp mappings.  This means @temp isn't
 * always a valid OSD set on return: in the "only primary_temp" case,
 * @temp will have its ->primary >= 0 but ->size == 0.
 */
static void get_temp_osds(struct ceph_osdmap *osdmap,
                          struct ceph_pg_pool_info *pi,
                          const struct ceph_pg *pgid,
                          struct ceph_osds *temp)
{
        struct ceph_pg_mapping *pg;
        int i;

        ceph_osds_init(temp);

        /* pg_temp? */
        pg = lookup_pg_mapping(&osdmap->pg_temp, pgid);
        if (pg) {
                for (i = 0; i < pg->pg_temp.len; i++) {
                        if (ceph_osd_is_down(osdmap, pg->pg_temp.osds[i])) {
                                if (ceph_can_shift_osds(pi))
                                        continue;

                                temp->osds[temp->size++] = CRUSH_ITEM_NONE;
                        } else {
                                temp->osds[temp->size++] = pg->pg_temp.osds[i];
                        }
                }

                /* apply pg_temp's primary */
                for (i = 0; i < temp->size; i++) {
                        if (temp->osds[i] != CRUSH_ITEM_NONE) {
                                temp->primary = temp->osds[i];
                                break;
                        }
                }
        }

        /* primary_temp? */
        pg = lookup_pg_mapping(&osdmap->primary_temp, pgid);
        if (pg)
                temp->primary = pg->primary_temp.osd;
}

/*
 * Map a PG to its acting set as well as its up set.
 *
 * Acting set is used for data mapping purposes, while up set can be
 * recorded for detecting interval changes and deciding whether to
 * resend a request.
 */
void ceph_pg_to_up_acting_osds(struct ceph_osdmap *osdmap,
                               struct ceph_pg_pool_info *pi,
                               const struct ceph_pg *raw_pgid,
                               struct ceph_osds *up,
                               struct ceph_osds *acting)
{
        struct ceph_pg pgid;
        u32 pps;

        WARN_ON(pi->id != raw_pgid->pool);
        raw_pg_to_pg(pi, raw_pgid, &pgid);

        pg_to_raw_osds(osdmap, pi, raw_pgid, up, &pps);
        apply_upmap(osdmap, &pgid, up);
        raw_to_up_osds(osdmap, pi, up);
        apply_primary_affinity(osdmap, pi, pps, up);
        get_temp_osds(osdmap, pi, &pgid, acting);
        if (!acting->size) {
                memcpy(acting->osds, up->osds, up->size * sizeof(up->osds[0]));
                acting->size = up->size;
                if (acting->primary == -1)
                        acting->primary = up->primary;
        }
        WARN_ON(!osds_valid(up) || !osds_valid(acting));
}

bool ceph_pg_to_primary_shard(struct ceph_osdmap *osdmap,
                              struct ceph_pg_pool_info *pi,
                              const struct ceph_pg *raw_pgid,
                              struct ceph_spg *spgid)
{
        struct ceph_pg pgid;
        struct ceph_osds up, acting;
        int i;

        WARN_ON(pi->id != raw_pgid->pool);
        raw_pg_to_pg(pi, raw_pgid, &pgid);

        if (ceph_can_shift_osds(pi)) {
                spgid->pgid = pgid; /* struct */
                spgid->shard = CEPH_SPG_NOSHARD;
                return true;
        }

        ceph_pg_to_up_acting_osds(osdmap, pi, &pgid, &up, &acting);
        for (i = 0; i < acting.size; i++) {
                if (acting.osds[i] == acting.primary) {
                        spgid->pgid = pgid; /* struct */
                        spgid->shard = i;
                        return true;
                }
        }

        return false;
}

/*
 * Return acting primary for given PG, or -1 if none.
 */
int ceph_pg_to_acting_primary(struct ceph_osdmap *osdmap,
                              const struct ceph_pg *raw_pgid)
{
        struct ceph_pg_pool_info *pi;
        struct ceph_osds up, acting;

        pi = ceph_pg_pool_by_id(osdmap, raw_pgid->pool);
        if (!pi)
                return -1;

        ceph_pg_to_up_acting_osds(osdmap, pi, raw_pgid, &up, &acting);
        return acting.primary;
}
EXPORT_SYMBOL(ceph_pg_to_acting_primary);

static struct crush_loc_node *alloc_crush_loc(size_t type_name_len,
                                              size_t name_len)
{
        struct crush_loc_node *loc;

        loc = kmalloc(sizeof(*loc) + type_name_len + name_len + 2, GFP_NOIO);
        if (!loc)
                return NULL;

        RB_CLEAR_NODE(&loc->cl_node);
        return loc;
}

static void free_crush_loc(struct crush_loc_node *loc)
{
        WARN_ON(!RB_EMPTY_NODE(&loc->cl_node));

        kfree(loc);
}

static int crush_loc_compare(const struct crush_loc *loc1,
                             const struct crush_loc *loc2)
{
        return strcmp(loc1->cl_type_name, loc2->cl_type_name) ?:
               strcmp(loc1->cl_name, loc2->cl_name);
}

DEFINE_RB_FUNCS2(crush_loc, struct crush_loc_node, cl_loc, crush_loc_compare,
                 RB_BYPTR, const struct crush_loc *, cl_node)

/*
 * Parses a set of <bucket type name>':'<bucket name> pairs separated
 * by '|', e.g. "rack:foo1|rack:foo2|datacenter:bar".
 *
 * Note that @crush_location is modified by strsep().
 */
int ceph_parse_crush_location(char *crush_location, struct rb_root *locs)
{
        struct crush_loc_node *loc;
        const char *type_name, *name, *colon;
        size_t type_name_len, name_len;

        dout("%s '%s'\n", __func__, crush_location);
        while ((type_name = strsep(&crush_location, "|"))) {
                colon = strchr(type_name, ':');
                if (!colon)
                        return -EINVAL;

                type_name_len = colon - type_name;
                if (type_name_len == 0)
                        return -EINVAL;

                name = colon + 1;
                name_len = strlen(name);
                if (name_len == 0)
                        return -EINVAL;

                loc = alloc_crush_loc(type_name_len, name_len);
                if (!loc)
                        return -ENOMEM;

                loc->cl_loc.cl_type_name = loc->cl_data;
                memcpy(loc->cl_loc.cl_type_name, type_name, type_name_len);
                loc->cl_loc.cl_type_name[type_name_len] = '\0';

                loc->cl_loc.cl_name = loc->cl_data + type_name_len + 1;
                memcpy(loc->cl_loc.cl_name, name, name_len);
                loc->cl_loc.cl_name[name_len] = '\0';

                if (!__insert_crush_loc(locs, loc)) {
                        free_crush_loc(loc);
                        return -EEXIST;
                }

                dout("%s type_name '%s' name '%s'\n", __func__,
                     loc->cl_loc.cl_type_name, loc->cl_loc.cl_name);
        }

        return 0;
}

int ceph_compare_crush_locs(struct rb_root *locs1, struct rb_root *locs2)
{
        struct rb_node *n1 = rb_first(locs1);
        struct rb_node *n2 = rb_first(locs2);
        int ret;

        for ( ; n1 && n2; n1 = rb_next(n1), n2 = rb_next(n2)) {
                struct crush_loc_node *loc1 =
                    rb_entry(n1, struct crush_loc_node, cl_node);
                struct crush_loc_node *loc2 =
                    rb_entry(n2, struct crush_loc_node, cl_node);

                ret = crush_loc_compare(&loc1->cl_loc, &loc2->cl_loc);
                if (ret)
                        return ret;
        }

        if (!n1 && n2)
                return -1;
        if (n1 && !n2)
                return 1;
        return 0;
}

void ceph_clear_crush_locs(struct rb_root *locs)
{
        while (!RB_EMPTY_ROOT(locs)) {
                struct crush_loc_node *loc =
                    rb_entry(rb_first(locs), struct crush_loc_node, cl_node);

                erase_crush_loc(locs, loc);
                free_crush_loc(loc);
        }
}

/*
 * [a-zA-Z0-9-_.]+
 */
static bool is_valid_crush_name(const char *name)
{
        do {
                if (!('a' <= *name && *name <= 'z') &&
                    !('A' <= *name && *name <= 'Z') &&
                    !('0' <= *name && *name <= '9') &&
                    *name != '-' && *name != '_' && *name != '.')
                        return false;
        } while (*++name != '\0');

        return true;
}

/*
 * Gets the parent of an item.  Returns its id (<0 because the
 * parent is always a bucket), type id (>0 for the same reason,
 * via @parent_type_id) and location (via @parent_loc).  If no
 * parent, returns 0.
 *
 * Does a linear search, as there are no parent pointers of any
 * kind.  Note that the result is ambiguous for items that occur
 * multiple times in the map.
 */
static int get_immediate_parent(struct crush_map *c, int id,
                                u16 *parent_type_id,
                                struct crush_loc *parent_loc)
{
        struct crush_bucket *b;
        struct crush_name_node *type_cn, *cn;
        int i, j;

        for (i = 0; i < c->max_buckets; i++) {
                b = c->buckets[i];
                if (!b)
                        continue;

                /* ignore per-class shadow hierarchy */
                cn = lookup_crush_name(&c->names, b->id);
                if (!cn || !is_valid_crush_name(cn->cn_name))
                        continue;

                for (j = 0; j < b->size; j++) {
                        if (b->items[j] != id)
                                continue;

                        *parent_type_id = b->type;
                        type_cn = lookup_crush_name(&c->type_names, b->type);
                        parent_loc->cl_type_name = type_cn->cn_name;
                        parent_loc->cl_name = cn->cn_name;
                        return b->id;
                }
        }

        return 0;  /* no parent */
}

/*
 * Calculates the locality/distance from an item to a client
 * location expressed in terms of CRUSH hierarchy as a set of
 * (bucket type name, bucket name) pairs.  Specifically, looks
 * for the lowest-valued bucket type for which the location of
 * @id matches one of the locations in @locs, so for standard
 * bucket types (host = 1, rack = 3, datacenter = 8, zone = 9)
 * a matching host is closer than a matching rack and a matching
 * data center is closer than a matching zone.
 *
 * Specifying multiple locations (a "multipath" location) such
 * as "rack=foo1 rack=foo2 datacenter=bar" is allowed -- @locs
 * is a multimap.  The locality will be:
 *
 * - 3 for OSDs in racks foo1 and foo2
 * - 8 for OSDs in data center bar
 * - -1 for all other OSDs
 *
 * The lowest possible bucket type is 1, so the best locality
 * for an OSD is 1 (i.e. a matching host).  Locality 0 would be
 * the OSD itself.
 */
int ceph_get_crush_locality(struct ceph_osdmap *osdmap, int id,
                            struct rb_root *locs)
{
        struct crush_loc loc;
        u16 type_id;

        /*
         * Instead of repeated get_immediate_parent() calls,
         * the location of @id could be obtained with a single
         * depth-first traversal.
         */
        for (;;) {
                id = get_immediate_parent(osdmap->crush, id, &type_id, &loc);
                if (id >= 0)
                        return -1;  /* not local */

                if (lookup_crush_loc(locs, &loc))
                        return type_id;
        }
}























    8 
















   45 
























   56 


   51 


   42 
    6 








   43 


    4 





   43 
   51 













   56 
    1 




















   58 







1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * sha512_base.h - core logic for SHA-512 implementations
 *
 * Copyright (C) 2015 Linaro Ltd <ard.biesheuvel@linaro.org>
 */

#ifndef _CRYPTO_SHA512_BASE_H
#define _CRYPTO_SHA512_BASE_H

#include <crypto/internal/hash.h>
#include <crypto/sha2.h>
#include <linux/crypto.h>
#include <linux/module.h>
#include <linux/string.h>

#include <asm/unaligned.h>

typedef void (sha512_block_fn)(struct sha512_state *sst, u8 const *src,
                               int blocks);

static inline int sha384_base_init(struct shash_desc *desc)
{
        struct sha512_state *sctx = shash_desc_ctx(desc);

        sctx->state[0] = SHA384_H0;
        sctx->state[1] = SHA384_H1;
        sctx->state[2] = SHA384_H2;
        sctx->state[3] = SHA384_H3;
        sctx->state[4] = SHA384_H4;
        sctx->state[5] = SHA384_H5;
        sctx->state[6] = SHA384_H6;
        sctx->state[7] = SHA384_H7;
        sctx->count[0] = sctx->count[1] = 0;

        return 0;
}

static inline int sha512_base_init(struct shash_desc *desc)
{
        struct sha512_state *sctx = shash_desc_ctx(desc);

        sctx->state[0] = SHA512_H0;
        sctx->state[1] = SHA512_H1;
        sctx->state[2] = SHA512_H2;
        sctx->state[3] = SHA512_H3;
        sctx->state[4] = SHA512_H4;
        sctx->state[5] = SHA512_H5;
        sctx->state[6] = SHA512_H6;
        sctx->state[7] = SHA512_H7;
        sctx->count[0] = sctx->count[1] = 0;

        return 0;
}

static inline int sha512_base_do_update(struct shash_desc *desc,
                                        const u8 *data,
                                        unsigned int len,
                                        sha512_block_fn *block_fn)
{
        struct sha512_state *sctx = shash_desc_ctx(desc);
        unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE;

        sctx->count[0] += len;
        if (sctx->count[0] < len)
                sctx->count[1]++;

        if (unlikely((partial + len) >= SHA512_BLOCK_SIZE)) {
                int blocks;

                if (partial) {
                        int p = SHA512_BLOCK_SIZE - partial;

                        memcpy(sctx->buf + partial, data, p);
                        data += p;
                        len -= p;

                        block_fn(sctx, sctx->buf, 1);
                }

                blocks = len / SHA512_BLOCK_SIZE;
                len %= SHA512_BLOCK_SIZE;

                if (blocks) {
                        block_fn(sctx, data, blocks);
                        data += blocks * SHA512_BLOCK_SIZE;
                }
                partial = 0;
        }
        if (len)
                memcpy(sctx->buf + partial, data, len);

        return 0;
}

static inline int sha512_base_do_finalize(struct shash_desc *desc,
                                          sha512_block_fn *block_fn)
{
        const int bit_offset = SHA512_BLOCK_SIZE - sizeof(__be64[2]);
        struct sha512_state *sctx = shash_desc_ctx(desc);
        __be64 *bits = (__be64 *)(sctx->buf + bit_offset);
        unsigned int partial = sctx->count[0] % SHA512_BLOCK_SIZE;

        sctx->buf[partial++] = 0x80;
        if (partial > bit_offset) {
                memset(sctx->buf + partial, 0x0, SHA512_BLOCK_SIZE - partial);
                partial = 0;

                block_fn(sctx, sctx->buf, 1);
        }

        memset(sctx->buf + partial, 0x0, bit_offset - partial);
        bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61);
        bits[1] = cpu_to_be64(sctx->count[0] << 3);
        block_fn(sctx, sctx->buf, 1);

        return 0;
}

static inline int sha512_base_finish(struct shash_desc *desc, u8 *out)
{
        unsigned int digest_size = crypto_shash_digestsize(desc->tfm);
        struct sha512_state *sctx = shash_desc_ctx(desc);
        __be64 *digest = (__be64 *)out;
        int i;

        for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be64))
                put_unaligned_be64(sctx->state[i], digest++);

        memzero_explicit(sctx, sizeof(*sctx));
        return 0;
}

#endif /* _CRYPTO_SHA512_BASE_H */


























  156 



















   77 











    2 

    2 









    3 

    3 









    9 


    9 





























































  150 





  151 

  151 










   86 


   86 

   87 



   86 












  104 

  103 


  103 












  104 



  104 










    2 

    2 



    2 



















































































































   59 



   59 

   59 
   58 








    7 

    8 
    8 


































    3 
    3 

    3 











    3 
    3 

    3 










    1 
    1 




    1 



























    1 
    1 


    1 

    1 

    1 











   77 
   77 


   76 

   76 

   77 












   51 
   51 


   51 











   51 
   50 



   51 











    1 
    1 


    1 












    1 
    1 


    1 


















   77 
   77 


   77 
   77 


   77 






















    9 
    9 


    8 
    9 


    9 










   83 
   83 


   83 
   82 

   83 
























    7 

    7 
    7 















    4 


    4 

    4 






















  122 


    1 


  120 

  122 
  121 




































































































    1 
    1 

    1 










   18 
   18 

   18 
































































































































































   18 

   18 
   18 

   18 













   74 

   74 
   74 






   23 



   23 

   23 


   23 

















   17 





   17 


   17 
   17 

   17 










    6 


    6 
    6 

    6 























    6 


    6 

    6 
    6 


    6 







    5 
    5 

    5 












    8 
















   15 


   15 


   15 
   15 


   15 

























































































   74 


   73 


   74 
   74 

   74 







    1 
    1 

    1 























































































   17 


   17 







   17 















































































































































































































































































































    5 
    5 
    5 


    5 





















































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Portions of this file
* Copyright(c) 2016 Intel Deutschland GmbH
* Copyright (C) 2018-2019, 2021-2024 Intel Corporation
*/

#ifndef __MAC80211_DRIVER_OPS
#define __MAC80211_DRIVER_OPS

#include <net/mac80211.h>
#include "ieee80211_i.h"
#include "trace.h"

#define check_sdata_in_driver(sdata)        ({                                        \
        WARN_ONCE(!sdata->local->reconfig_failure &&                                \
                  !(sdata->flags & IEEE80211_SDATA_IN_DRIVER),                        \
                  "%s: Failed check-sdata-in-driver check, flags: 0x%x\n",        \
                  sdata->dev ? sdata->dev->name : sdata->name, sdata->flags);        \
        !!(sdata->flags & IEEE80211_SDATA_IN_DRIVER);                                \
})

static inline struct ieee80211_sub_if_data *
get_bss_sdata(struct ieee80211_sub_if_data *sdata)
{
        if (sdata && sdata->vif.type == NL80211_IFTYPE_AP_VLAN)
                sdata = container_of(sdata->bss, struct ieee80211_sub_if_data,
                                     u.ap);

        return sdata;
}

static inline void drv_tx(struct ieee80211_local *local,
                          struct ieee80211_tx_control *control,
                          struct sk_buff *skb)
{
        local->ops->tx(&local->hw, control, skb);
}

static inline void drv_sync_rx_queues(struct ieee80211_local *local,
                                      struct sta_info *sta)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (local->ops->sync_rx_queues) {
                trace_drv_sync_rx_queues(local, sta->sdata, &sta->sta);
                local->ops->sync_rx_queues(&local->hw);
                trace_drv_return_void(local);
        }
}

static inline void drv_get_et_strings(struct ieee80211_sub_if_data *sdata,
                                      u32 sset, u8 *data)
{
        struct ieee80211_local *local = sdata->local;
        if (local->ops->get_et_strings) {
                trace_drv_get_et_strings(local, sset);
                local->ops->get_et_strings(&local->hw, &sdata->vif, sset, data);
                trace_drv_return_void(local);
        }
}

static inline void drv_get_et_stats(struct ieee80211_sub_if_data *sdata,
                                    struct ethtool_stats *stats,
                                    u64 *data)
{
        struct ieee80211_local *local = sdata->local;
        if (local->ops->get_et_stats) {
                trace_drv_get_et_stats(local);
                local->ops->get_et_stats(&local->hw, &sdata->vif, stats, data);
                trace_drv_return_void(local);
        }
}

static inline int drv_get_et_sset_count(struct ieee80211_sub_if_data *sdata,
                                        int sset)
{
        struct ieee80211_local *local = sdata->local;
        int rv = 0;
        if (local->ops->get_et_sset_count) {
                trace_drv_get_et_sset_count(local, sset);
                rv = local->ops->get_et_sset_count(&local->hw, &sdata->vif,
                                                   sset);
                trace_drv_return_int(local, rv);
        }
        return rv;
}

int drv_start(struct ieee80211_local *local);
void drv_stop(struct ieee80211_local *local, bool suspend);

#ifdef CONFIG_PM
static inline int drv_suspend(struct ieee80211_local *local,
                              struct cfg80211_wowlan *wowlan)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_suspend(local);
        ret = local->ops->suspend(&local->hw, wowlan);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline int drv_resume(struct ieee80211_local *local)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_resume(local);
        ret = local->ops->resume(&local->hw);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline void drv_set_wakeup(struct ieee80211_local *local,
                                  bool enabled)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!local->ops->set_wakeup)
                return;

        trace_drv_set_wakeup(local, enabled);
        local->ops->set_wakeup(&local->hw, enabled);
        trace_drv_return_void(local);
}
#endif

int drv_add_interface(struct ieee80211_local *local,
                      struct ieee80211_sub_if_data *sdata);

int drv_change_interface(struct ieee80211_local *local,
                         struct ieee80211_sub_if_data *sdata,
                         enum nl80211_iftype type, bool p2p);

void drv_remove_interface(struct ieee80211_local *local,
                          struct ieee80211_sub_if_data *sdata);

static inline int drv_config(struct ieee80211_local *local, u32 changed)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_config(local, changed);
        ret = local->ops->config(&local->hw, changed);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline void drv_vif_cfg_changed(struct ieee80211_local *local,
                                       struct ieee80211_sub_if_data *sdata,
                                       u64 changed)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_vif_cfg_changed(local, sdata, changed);
        if (local->ops->vif_cfg_changed)
                local->ops->vif_cfg_changed(&local->hw, &sdata->vif, changed);
        else if (local->ops->bss_info_changed)
                local->ops->bss_info_changed(&local->hw, &sdata->vif,
                                             &sdata->vif.bss_conf, changed);
        trace_drv_return_void(local);
}

void drv_link_info_changed(struct ieee80211_local *local,
                           struct ieee80211_sub_if_data *sdata,
                           struct ieee80211_bss_conf *info,
                           int link_id, u64 changed);

static inline u64 drv_prepare_multicast(struct ieee80211_local *local,
                                        struct netdev_hw_addr_list *mc_list)
{
        u64 ret = 0;

        trace_drv_prepare_multicast(local, mc_list->count);

        if (local->ops->prepare_multicast)
                ret = local->ops->prepare_multicast(&local->hw, mc_list);

        trace_drv_return_u64(local, ret);

        return ret;
}

static inline void drv_configure_filter(struct ieee80211_local *local,
                                        unsigned int changed_flags,
                                        unsigned int *total_flags,
                                        u64 multicast)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_configure_filter(local, changed_flags, total_flags,
                                   multicast);
        local->ops->configure_filter(&local->hw, changed_flags, total_flags,
                                     multicast);
        trace_drv_return_void(local);
}

static inline void drv_config_iface_filter(struct ieee80211_local *local,
                                           struct ieee80211_sub_if_data *sdata,
                                           unsigned int filter_flags,
                                           unsigned int changed_flags)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_config_iface_filter(local, sdata, filter_flags,
                                      changed_flags);
        if (local->ops->config_iface_filter)
                local->ops->config_iface_filter(&local->hw, &sdata->vif,
                                                filter_flags,
                                                changed_flags);
        trace_drv_return_void(local);
}

static inline int drv_set_tim(struct ieee80211_local *local,
                              struct ieee80211_sta *sta, bool set)
{
        int ret = 0;
        trace_drv_set_tim(local, sta, set);
        if (local->ops->set_tim)
                ret = local->ops->set_tim(&local->hw, sta, set);
        trace_drv_return_int(local, ret);
        return ret;
}

int drv_set_key(struct ieee80211_local *local,
                enum set_key_cmd cmd,
                struct ieee80211_sub_if_data *sdata,
                struct ieee80211_sta *sta,
                struct ieee80211_key_conf *key);

static inline void drv_update_tkip_key(struct ieee80211_local *local,
                                       struct ieee80211_sub_if_data *sdata,
                                       struct ieee80211_key_conf *conf,
                                       struct sta_info *sta, u32 iv32,
                                       u16 *phase1key)
{
        struct ieee80211_sta *ista = NULL;

        if (sta)
                ista = &sta->sta;

        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_update_tkip_key(local, sdata, conf, ista, iv32);
        if (local->ops->update_tkip_key)
                local->ops->update_tkip_key(&local->hw, &sdata->vif, conf,
                                            ista, iv32, phase1key);
        trace_drv_return_void(local);
}

static inline int drv_hw_scan(struct ieee80211_local *local,
                              struct ieee80211_sub_if_data *sdata,
                              struct ieee80211_scan_request *req)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return -EIO;

        trace_drv_hw_scan(local, sdata);
        ret = local->ops->hw_scan(&local->hw, &sdata->vif, req);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline void drv_cancel_hw_scan(struct ieee80211_local *local,
                                      struct ieee80211_sub_if_data *sdata)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_cancel_hw_scan(local, sdata);
        local->ops->cancel_hw_scan(&local->hw, &sdata->vif);
        trace_drv_return_void(local);
}

static inline int
drv_sched_scan_start(struct ieee80211_local *local,
                     struct ieee80211_sub_if_data *sdata,
                     struct cfg80211_sched_scan_request *req,
                     struct ieee80211_scan_ies *ies)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return -EIO;

        trace_drv_sched_scan_start(local, sdata);
        ret = local->ops->sched_scan_start(&local->hw, &sdata->vif,
                                              req, ies);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline int drv_sched_scan_stop(struct ieee80211_local *local,
                                      struct ieee80211_sub_if_data *sdata)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return -EIO;

        trace_drv_sched_scan_stop(local, sdata);
        ret = local->ops->sched_scan_stop(&local->hw, &sdata->vif);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline void drv_sw_scan_start(struct ieee80211_local *local,
                                     struct ieee80211_sub_if_data *sdata,
                                     const u8 *mac_addr)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_sw_scan_start(local, sdata, mac_addr);
        if (local->ops->sw_scan_start)
                local->ops->sw_scan_start(&local->hw, &sdata->vif, mac_addr);
        trace_drv_return_void(local);
}

static inline void drv_sw_scan_complete(struct ieee80211_local *local,
                                        struct ieee80211_sub_if_data *sdata)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_sw_scan_complete(local, sdata);
        if (local->ops->sw_scan_complete)
                local->ops->sw_scan_complete(&local->hw, &sdata->vif);
        trace_drv_return_void(local);
}

static inline int drv_get_stats(struct ieee80211_local *local,
                                struct ieee80211_low_level_stats *stats)
{
        int ret = -EOPNOTSUPP;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (local->ops->get_stats)
                ret = local->ops->get_stats(&local->hw, stats);
        trace_drv_get_stats(local, stats, ret);

        return ret;
}

static inline void drv_get_key_seq(struct ieee80211_local *local,
                                   struct ieee80211_key *key,
                                   struct ieee80211_key_seq *seq)
{
        if (local->ops->get_key_seq)
                local->ops->get_key_seq(&local->hw, &key->conf, seq);
        trace_drv_get_key_seq(local, &key->conf);
}

static inline int drv_set_frag_threshold(struct ieee80211_local *local,
                                        u32 value)
{
        int ret = 0;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_set_frag_threshold(local, value);
        if (local->ops->set_frag_threshold)
                ret = local->ops->set_frag_threshold(&local->hw, value);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline int drv_set_rts_threshold(struct ieee80211_local *local,
                                        u32 value)
{
        int ret = 0;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_set_rts_threshold(local, value);
        if (local->ops->set_rts_threshold)
                ret = local->ops->set_rts_threshold(&local->hw, value);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline int drv_set_coverage_class(struct ieee80211_local *local,
                                         s16 value)
{
        int ret = 0;
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_set_coverage_class(local, value);
        if (local->ops->set_coverage_class)
                local->ops->set_coverage_class(&local->hw, value);
        else
                ret = -EOPNOTSUPP;

        trace_drv_return_int(local, ret);
        return ret;
}

static inline void drv_sta_notify(struct ieee80211_local *local,
                                  struct ieee80211_sub_if_data *sdata,
                                  enum sta_notify_cmd cmd,
                                  struct ieee80211_sta *sta)
{
        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_sta_notify(local, sdata, cmd, sta);
        if (local->ops->sta_notify)
                local->ops->sta_notify(&local->hw, &sdata->vif, cmd, sta);
        trace_drv_return_void(local);
}

static inline int drv_sta_add(struct ieee80211_local *local,
                              struct ieee80211_sub_if_data *sdata,
                              struct ieee80211_sta *sta)
{
        int ret = 0;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return -EIO;

        trace_drv_sta_add(local, sdata, sta);
        if (local->ops->sta_add)
                ret = local->ops->sta_add(&local->hw, &sdata->vif, sta);

        trace_drv_return_int(local, ret);

        return ret;
}

static inline void drv_sta_remove(struct ieee80211_local *local,
                                  struct ieee80211_sub_if_data *sdata,
                                  struct ieee80211_sta *sta)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_sta_remove(local, sdata, sta);
        if (local->ops->sta_remove)
                local->ops->sta_remove(&local->hw, &sdata->vif, sta);

        trace_drv_return_void(local);
}

#ifdef CONFIG_MAC80211_DEBUGFS
static inline void drv_vif_add_debugfs(struct ieee80211_local *local,
                                       struct ieee80211_sub_if_data *sdata)
{
        might_sleep();

        if (sdata->vif.type == NL80211_IFTYPE_MONITOR ||
            WARN_ON(!sdata->vif.debugfs_dir))
                return;

        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return;

        if (local->ops->vif_add_debugfs)
                local->ops->vif_add_debugfs(&local->hw, &sdata->vif);
}

static inline void drv_link_add_debugfs(struct ieee80211_local *local,
                                        struct ieee80211_sub_if_data *sdata,
                                        struct ieee80211_bss_conf *link_conf,
                                        struct dentry *dir)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return;

        if (local->ops->link_add_debugfs)
                local->ops->link_add_debugfs(&local->hw, &sdata->vif,
                                             link_conf, dir);
}

static inline void drv_sta_add_debugfs(struct ieee80211_local *local,
                                       struct ieee80211_sub_if_data *sdata,
                                       struct ieee80211_sta *sta,
                                       struct dentry *dir)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return;

        if (local->ops->sta_add_debugfs)
                local->ops->sta_add_debugfs(&local->hw, &sdata->vif,
                                            sta, dir);
}

static inline void drv_link_sta_add_debugfs(struct ieee80211_local *local,
                                            struct ieee80211_sub_if_data *sdata,
                                            struct ieee80211_link_sta *link_sta,
                                            struct dentry *dir)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return;

        if (local->ops->link_sta_add_debugfs)
                local->ops->link_sta_add_debugfs(&local->hw, &sdata->vif,
                                                 link_sta, dir);
}
#else
static inline void drv_vif_add_debugfs(struct ieee80211_local *local,
                                       struct ieee80211_sub_if_data *sdata)
{
        might_sleep();
}
#endif

static inline void drv_sta_pre_rcu_remove(struct ieee80211_local *local,
                                          struct ieee80211_sub_if_data *sdata,
                                          struct sta_info *sta)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_sta_pre_rcu_remove(local, sdata, &sta->sta);
        if (local->ops->sta_pre_rcu_remove)
                local->ops->sta_pre_rcu_remove(&local->hw, &sdata->vif,
                                               &sta->sta);
        trace_drv_return_void(local);
}

__must_check
int drv_sta_state(struct ieee80211_local *local,
                  struct ieee80211_sub_if_data *sdata,
                  struct sta_info *sta,
                  enum ieee80211_sta_state old_state,
                  enum ieee80211_sta_state new_state);

__must_check
int drv_sta_set_txpwr(struct ieee80211_local *local,
                      struct ieee80211_sub_if_data *sdata,
                      struct sta_info *sta);

void drv_sta_rc_update(struct ieee80211_local *local,
                       struct ieee80211_sub_if_data *sdata,
                       struct ieee80211_sta *sta, u32 changed);

static inline void drv_sta_rate_tbl_update(struct ieee80211_local *local,
                                           struct ieee80211_sub_if_data *sdata,
                                           struct ieee80211_sta *sta)
{
        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_sta_rate_tbl_update(local, sdata, sta);
        if (local->ops->sta_rate_tbl_update)
                local->ops->sta_rate_tbl_update(&local->hw, &sdata->vif, sta);

        trace_drv_return_void(local);
}

static inline void drv_sta_statistics(struct ieee80211_local *local,
                                      struct ieee80211_sub_if_data *sdata,
                                      struct ieee80211_sta *sta,
                                      struct station_info *sinfo)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_sta_statistics(local, sdata, sta);
        if (local->ops->sta_statistics)
                local->ops->sta_statistics(&local->hw, &sdata->vif, sta, sinfo);
        trace_drv_return_void(local);
}

int drv_conf_tx(struct ieee80211_local *local,
                struct ieee80211_link_data *link, u16 ac,
                const struct ieee80211_tx_queue_params *params);

u64 drv_get_tsf(struct ieee80211_local *local,
                struct ieee80211_sub_if_data *sdata);
void drv_set_tsf(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 u64 tsf);
void drv_offset_tsf(struct ieee80211_local *local,
                    struct ieee80211_sub_if_data *sdata,
                    s64 offset);
void drv_reset_tsf(struct ieee80211_local *local,
                   struct ieee80211_sub_if_data *sdata);

static inline int drv_tx_last_beacon(struct ieee80211_local *local)
{
        int ret = 0; /* default unsupported op for less congestion */

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_tx_last_beacon(local);
        if (local->ops->tx_last_beacon)
                ret = local->ops->tx_last_beacon(&local->hw);
        trace_drv_return_int(local, ret);
        return ret;
}

int drv_ampdu_action(struct ieee80211_local *local,
                     struct ieee80211_sub_if_data *sdata,
                     struct ieee80211_ampdu_params *params);

static inline int drv_get_survey(struct ieee80211_local *local, int idx,
                                struct survey_info *survey)
{
        int ret = -EOPNOTSUPP;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_get_survey(local, idx, survey);

        if (local->ops->get_survey)
                ret = local->ops->get_survey(&local->hw, idx, survey);

        trace_drv_return_int(local, ret);

        return ret;
}

static inline void drv_rfkill_poll(struct ieee80211_local *local)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (local->ops->rfkill_poll)
                local->ops->rfkill_poll(&local->hw);
}

static inline void drv_flush(struct ieee80211_local *local,
                             struct ieee80211_sub_if_data *sdata,
                             u32 queues, bool drop)
{
        struct ieee80211_vif *vif;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        sdata = get_bss_sdata(sdata);
        vif = sdata ? &sdata->vif : NULL;

        if (sdata && !check_sdata_in_driver(sdata))
                return;

        trace_drv_flush(local, queues, drop);
        if (local->ops->flush)
                local->ops->flush(&local->hw, vif, queues, drop);
        trace_drv_return_void(local);
}

static inline void drv_flush_sta(struct ieee80211_local *local,
                                 struct ieee80211_sub_if_data *sdata,
                                 struct sta_info *sta)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        sdata = get_bss_sdata(sdata);

        if (sdata && !check_sdata_in_driver(sdata))
                return;

        trace_drv_flush_sta(local, sdata, &sta->sta);
        if (local->ops->flush_sta)
                local->ops->flush_sta(&local->hw, &sdata->vif, &sta->sta);
        trace_drv_return_void(local);
}

static inline void drv_channel_switch(struct ieee80211_local *local,
                                      struct ieee80211_sub_if_data *sdata,
                                      struct ieee80211_channel_switch *ch_switch)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_channel_switch(local, sdata, ch_switch);
        local->ops->channel_switch(&local->hw, &sdata->vif, ch_switch);
        trace_drv_return_void(local);
}


static inline int drv_set_antenna(struct ieee80211_local *local,
                                  u32 tx_ant, u32 rx_ant)
{
        int ret = -EOPNOTSUPP;
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (local->ops->set_antenna)
                ret = local->ops->set_antenna(&local->hw, tx_ant, rx_ant);
        trace_drv_set_antenna(local, tx_ant, rx_ant, ret);
        return ret;
}

static inline int drv_get_antenna(struct ieee80211_local *local,
                                  u32 *tx_ant, u32 *rx_ant)
{
        int ret = -EOPNOTSUPP;
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (local->ops->get_antenna)
                ret = local->ops->get_antenna(&local->hw, tx_ant, rx_ant);
        trace_drv_get_antenna(local, *tx_ant, *rx_ant, ret);
        return ret;
}

static inline int drv_remain_on_channel(struct ieee80211_local *local,
                                        struct ieee80211_sub_if_data *sdata,
                                        struct ieee80211_channel *chan,
                                        unsigned int duration,
                                        enum ieee80211_roc_type type)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_remain_on_channel(local, sdata, chan, duration, type);
        ret = local->ops->remain_on_channel(&local->hw, &sdata->vif,
                                            chan, duration, type);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline int
drv_cancel_remain_on_channel(struct ieee80211_local *local,
                             struct ieee80211_sub_if_data *sdata)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_cancel_remain_on_channel(local, sdata);
        ret = local->ops->cancel_remain_on_channel(&local->hw, &sdata->vif);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline int drv_set_ringparam(struct ieee80211_local *local,
                                    u32 tx, u32 rx)
{
        int ret = -EOPNOTSUPP;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_set_ringparam(local, tx, rx);
        if (local->ops->set_ringparam)
                ret = local->ops->set_ringparam(&local->hw, tx, rx);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline void drv_get_ringparam(struct ieee80211_local *local,
                                     u32 *tx, u32 *tx_max, u32 *rx, u32 *rx_max)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_get_ringparam(local, tx, tx_max, rx, rx_max);
        if (local->ops->get_ringparam)
                local->ops->get_ringparam(&local->hw, tx, tx_max, rx, rx_max);
        trace_drv_return_void(local);
}

static inline bool drv_tx_frames_pending(struct ieee80211_local *local)
{
        bool ret = false;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_tx_frames_pending(local);
        if (local->ops->tx_frames_pending)
                ret = local->ops->tx_frames_pending(&local->hw);
        trace_drv_return_bool(local, ret);

        return ret;
}

static inline int drv_set_bitrate_mask(struct ieee80211_local *local,
                                       struct ieee80211_sub_if_data *sdata,
                                       const struct cfg80211_bitrate_mask *mask)
{
        int ret = -EOPNOTSUPP;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return -EIO;

        trace_drv_set_bitrate_mask(local, sdata, mask);
        if (local->ops->set_bitrate_mask)
                ret = local->ops->set_bitrate_mask(&local->hw,
                                                   &sdata->vif, mask);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline void drv_set_rekey_data(struct ieee80211_local *local,
                                      struct ieee80211_sub_if_data *sdata,
                                      struct cfg80211_gtk_rekey_data *data)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_set_rekey_data(local, sdata, data);
        if (local->ops->set_rekey_data)
                local->ops->set_rekey_data(&local->hw, &sdata->vif, data);
        trace_drv_return_void(local);
}

static inline void drv_event_callback(struct ieee80211_local *local,
                                      struct ieee80211_sub_if_data *sdata,
                                      const struct ieee80211_event *event)
{
        trace_drv_event_callback(local, sdata, event);
        if (local->ops->event_callback)
                local->ops->event_callback(&local->hw, &sdata->vif, event);
        trace_drv_return_void(local);
}

static inline void
drv_release_buffered_frames(struct ieee80211_local *local,
                            struct sta_info *sta, u16 tids, int num_frames,
                            enum ieee80211_frame_release_type reason,
                            bool more_data)
{
        trace_drv_release_buffered_frames(local, &sta->sta, tids, num_frames,
                                          reason, more_data);
        if (local->ops->release_buffered_frames)
                local->ops->release_buffered_frames(&local->hw, &sta->sta, tids,
                                                    num_frames, reason,
                                                    more_data);
        trace_drv_return_void(local);
}

static inline void
drv_allow_buffered_frames(struct ieee80211_local *local,
                          struct sta_info *sta, u16 tids, int num_frames,
                          enum ieee80211_frame_release_type reason,
                          bool more_data)
{
        trace_drv_allow_buffered_frames(local, &sta->sta, tids, num_frames,
                                        reason, more_data);
        if (local->ops->allow_buffered_frames)
                local->ops->allow_buffered_frames(&local->hw, &sta->sta,
                                                  tids, num_frames, reason,
                                                  more_data);
        trace_drv_return_void(local);
}

static inline void drv_mgd_prepare_tx(struct ieee80211_local *local,
                                      struct ieee80211_sub_if_data *sdata,
                                      struct ieee80211_prep_tx_info *info)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return;
        WARN_ON_ONCE(sdata->vif.type != NL80211_IFTYPE_STATION);

        info->link_id = info->link_id < 0 ? 0 : info->link_id;
        trace_drv_mgd_prepare_tx(local, sdata, info->duration,
                                 info->subtype, info->success);
        if (local->ops->mgd_prepare_tx)
                local->ops->mgd_prepare_tx(&local->hw, &sdata->vif, info);
        trace_drv_return_void(local);
}

static inline void drv_mgd_complete_tx(struct ieee80211_local *local,
                                       struct ieee80211_sub_if_data *sdata,
                                       struct ieee80211_prep_tx_info *info)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return;
        WARN_ON_ONCE(sdata->vif.type != NL80211_IFTYPE_STATION);

        trace_drv_mgd_complete_tx(local, sdata, info->duration,
                                  info->subtype, info->success);
        if (local->ops->mgd_complete_tx)
                local->ops->mgd_complete_tx(&local->hw, &sdata->vif, info);
        trace_drv_return_void(local);
}

static inline void
drv_mgd_protect_tdls_discover(struct ieee80211_local *local,
                              struct ieee80211_sub_if_data *sdata,
                              int link_id)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return;
        WARN_ON_ONCE(sdata->vif.type != NL80211_IFTYPE_STATION);

        link_id = link_id > 0 ? link_id : 0;

        trace_drv_mgd_protect_tdls_discover(local, sdata);
        if (local->ops->mgd_protect_tdls_discover)
                local->ops->mgd_protect_tdls_discover(&local->hw, &sdata->vif,
                                                      link_id);
        trace_drv_return_void(local);
}

static inline int drv_add_chanctx(struct ieee80211_local *local,
                                  struct ieee80211_chanctx *ctx)
{
        int ret = -EOPNOTSUPP;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_add_chanctx(local, ctx);
        if (local->ops->add_chanctx)
                ret = local->ops->add_chanctx(&local->hw, &ctx->conf);
        trace_drv_return_int(local, ret);
        if (!ret)
                ctx->driver_present = true;

        return ret;
}

static inline void drv_remove_chanctx(struct ieee80211_local *local,
                                      struct ieee80211_chanctx *ctx)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (WARN_ON(!ctx->driver_present))
                return;

        trace_drv_remove_chanctx(local, ctx);
        if (local->ops->remove_chanctx)
                local->ops->remove_chanctx(&local->hw, &ctx->conf);
        trace_drv_return_void(local);
        ctx->driver_present = false;
}

static inline void drv_change_chanctx(struct ieee80211_local *local,
                                      struct ieee80211_chanctx *ctx,
                                      u32 changed)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_change_chanctx(local, ctx, changed);
        if (local->ops->change_chanctx) {
                WARN_ON_ONCE(!ctx->driver_present);
                local->ops->change_chanctx(&local->hw, &ctx->conf, changed);
        }
        trace_drv_return_void(local);
}

int drv_assign_vif_chanctx(struct ieee80211_local *local,
                           struct ieee80211_sub_if_data *sdata,
                           struct ieee80211_bss_conf *link_conf,
                           struct ieee80211_chanctx *ctx);
void drv_unassign_vif_chanctx(struct ieee80211_local *local,
                              struct ieee80211_sub_if_data *sdata,
                              struct ieee80211_bss_conf *link_conf,
                              struct ieee80211_chanctx *ctx);
int drv_switch_vif_chanctx(struct ieee80211_local *local,
                           struct ieee80211_vif_chanctx_switch *vifs,
                           int n_vifs, enum ieee80211_chanctx_switch_mode mode);

static inline int drv_start_ap(struct ieee80211_local *local,
                               struct ieee80211_sub_if_data *sdata,
                               struct ieee80211_bss_conf *link_conf)
{
        int ret = 0;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return -EIO;

        trace_drv_start_ap(local, sdata, link_conf);
        if (local->ops->start_ap)
                ret = local->ops->start_ap(&local->hw, &sdata->vif, link_conf);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline void drv_stop_ap(struct ieee80211_local *local,
                               struct ieee80211_sub_if_data *sdata,
                               struct ieee80211_bss_conf *link_conf)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_stop_ap(local, sdata, link_conf);
        if (local->ops->stop_ap)
                local->ops->stop_ap(&local->hw, &sdata->vif, link_conf);
        trace_drv_return_void(local);
}

static inline void
drv_reconfig_complete(struct ieee80211_local *local,
                      enum ieee80211_reconfig_type reconfig_type)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        trace_drv_reconfig_complete(local, reconfig_type);
        if (local->ops->reconfig_complete)
                local->ops->reconfig_complete(&local->hw, reconfig_type);
        trace_drv_return_void(local);
}

static inline void
drv_set_default_unicast_key(struct ieee80211_local *local,
                            struct ieee80211_sub_if_data *sdata,
                            int key_idx)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return;

        WARN_ON_ONCE(key_idx < -1 || key_idx > 3);

        trace_drv_set_default_unicast_key(local, sdata, key_idx);
        if (local->ops->set_default_unicast_key)
                local->ops->set_default_unicast_key(&local->hw, &sdata->vif,
                                                    key_idx);
        trace_drv_return_void(local);
}

#if IS_ENABLED(CONFIG_IPV6)
static inline void drv_ipv6_addr_change(struct ieee80211_local *local,
                                        struct ieee80211_sub_if_data *sdata,
                                        struct inet6_dev *idev)
{
        trace_drv_ipv6_addr_change(local, sdata);
        if (local->ops->ipv6_addr_change)
                local->ops->ipv6_addr_change(&local->hw, &sdata->vif, idev);
        trace_drv_return_void(local);
}
#endif

static inline void
drv_channel_switch_beacon(struct ieee80211_sub_if_data *sdata,
                          struct cfg80211_chan_def *chandef)
{
        struct ieee80211_local *local = sdata->local;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (local->ops->channel_switch_beacon) {
                trace_drv_channel_switch_beacon(local, sdata, chandef);
                local->ops->channel_switch_beacon(&local->hw, &sdata->vif,
                                                  chandef);
        }
}

static inline int
drv_pre_channel_switch(struct ieee80211_sub_if_data *sdata,
                       struct ieee80211_channel_switch *ch_switch)
{
        struct ieee80211_local *local = sdata->local;
        int ret = 0;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return -EIO;

        if (!ieee80211_vif_link_active(&sdata->vif, ch_switch->link_id))
                return 0;

        trace_drv_pre_channel_switch(local, sdata, ch_switch);
        if (local->ops->pre_channel_switch)
                ret = local->ops->pre_channel_switch(&local->hw, &sdata->vif,
                                                     ch_switch);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline int
drv_post_channel_switch(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_local *local = sdata->local;
        int ret = 0;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return -EIO;

        if (!ieee80211_vif_link_active(&sdata->vif, link->link_id))
                return 0;

        trace_drv_post_channel_switch(local, sdata);
        if (local->ops->post_channel_switch)
                ret = local->ops->post_channel_switch(&local->hw, &sdata->vif,
                                                      link->conf);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline void
drv_abort_channel_switch(struct ieee80211_link_data *link)
{
        struct ieee80211_sub_if_data *sdata = link->sdata;
        struct ieee80211_local *local = sdata->local;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return;

        if (!ieee80211_vif_link_active(&sdata->vif, link->link_id))
                return;

        trace_drv_abort_channel_switch(local, sdata);

        if (local->ops->abort_channel_switch)
                local->ops->abort_channel_switch(&local->hw, &sdata->vif,
                                                 link->conf);
}

static inline void
drv_channel_switch_rx_beacon(struct ieee80211_sub_if_data *sdata,
                             struct ieee80211_channel_switch *ch_switch)
{
        struct ieee80211_local *local = sdata->local;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return;

        if (!ieee80211_vif_link_active(&sdata->vif, ch_switch->link_id))
                return;

        trace_drv_channel_switch_rx_beacon(local, sdata, ch_switch);
        if (local->ops->channel_switch_rx_beacon)
                local->ops->channel_switch_rx_beacon(&local->hw, &sdata->vif,
                                                     ch_switch);
}

static inline int drv_join_ibss(struct ieee80211_local *local,
                                struct ieee80211_sub_if_data *sdata)
{
        int ret = 0;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (!check_sdata_in_driver(sdata))
                return -EIO;

        trace_drv_join_ibss(local, sdata, &sdata->vif.bss_conf);
        if (local->ops->join_ibss)
                ret = local->ops->join_ibss(&local->hw, &sdata->vif);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline void drv_leave_ibss(struct ieee80211_local *local,
                                  struct ieee80211_sub_if_data *sdata)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_leave_ibss(local, sdata);
        if (local->ops->leave_ibss)
                local->ops->leave_ibss(&local->hw, &sdata->vif);
        trace_drv_return_void(local);
}

static inline u32 drv_get_expected_throughput(struct ieee80211_local *local,
                                              struct sta_info *sta)
{
        u32 ret = 0;

        trace_drv_get_expected_throughput(&sta->sta);
        if (local->ops->get_expected_throughput && sta->uploaded)
                ret = local->ops->get_expected_throughput(&local->hw, &sta->sta);
        trace_drv_return_u32(local, ret);

        return ret;
}

static inline int drv_get_txpower(struct ieee80211_local *local,
                                  struct ieee80211_sub_if_data *sdata, int *dbm)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!local->ops->get_txpower)
                return -EOPNOTSUPP;

        ret = local->ops->get_txpower(&local->hw, &sdata->vif, dbm);
        trace_drv_get_txpower(local, sdata, *dbm, ret);

        return ret;
}

static inline int
drv_tdls_channel_switch(struct ieee80211_local *local,
                        struct ieee80211_sub_if_data *sdata,
                        struct ieee80211_sta *sta, u8 oper_class,
                        struct cfg80211_chan_def *chandef,
                        struct sk_buff *tmpl_skb, u32 ch_sw_tm_ie)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (!check_sdata_in_driver(sdata))
                return -EIO;

        if (!local->ops->tdls_channel_switch)
                return -EOPNOTSUPP;

        trace_drv_tdls_channel_switch(local, sdata, sta, oper_class, chandef);
        ret = local->ops->tdls_channel_switch(&local->hw, &sdata->vif, sta,
                                              oper_class, chandef, tmpl_skb,
                                              ch_sw_tm_ie);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline void
drv_tdls_cancel_channel_switch(struct ieee80211_local *local,
                               struct ieee80211_sub_if_data *sdata,
                               struct ieee80211_sta *sta)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (!check_sdata_in_driver(sdata))
                return;

        if (!local->ops->tdls_cancel_channel_switch)
                return;

        trace_drv_tdls_cancel_channel_switch(local, sdata, sta);
        local->ops->tdls_cancel_channel_switch(&local->hw, &sdata->vif, sta);
        trace_drv_return_void(local);
}

static inline void
drv_tdls_recv_channel_switch(struct ieee80211_local *local,
                             struct ieee80211_sub_if_data *sdata,
                             struct ieee80211_tdls_ch_sw_params *params)
{
        trace_drv_tdls_recv_channel_switch(local, sdata, params);
        if (local->ops->tdls_recv_channel_switch)
                local->ops->tdls_recv_channel_switch(&local->hw, &sdata->vif,
                                                     params);
        trace_drv_return_void(local);
}

static inline void drv_wake_tx_queue(struct ieee80211_local *local,
                                     struct txq_info *txq)
{
        struct ieee80211_sub_if_data *sdata = vif_to_sdata(txq->txq.vif);

        /* In reconfig don't transmit now, but mark for waking later */
        if (local->in_reconfig) {
                set_bit(IEEE80211_TXQ_DIRTY, &txq->flags);
                return;
        }

        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_wake_tx_queue(local, sdata, txq);
        local->ops->wake_tx_queue(&local->hw, &txq->txq);
}

static inline void schedule_and_wake_txq(struct ieee80211_local *local,
                                         struct txq_info *txqi)
{
        ieee80211_schedule_txq(&local->hw, &txqi->txq);
        drv_wake_tx_queue(local, txqi);
}

static inline int drv_can_aggregate_in_amsdu(struct ieee80211_local *local,
                                             struct sk_buff *head,
                                             struct sk_buff *skb)
{
        if (!local->ops->can_aggregate_in_amsdu)
                return true;

        return local->ops->can_aggregate_in_amsdu(&local->hw, head, skb);
}

static inline int
drv_get_ftm_responder_stats(struct ieee80211_local *local,
                            struct ieee80211_sub_if_data *sdata,
                            struct cfg80211_ftm_responder_stats *ftm_stats)
{
        u32 ret = -EOPNOTSUPP;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (!check_sdata_in_driver(sdata))
                return -EIO;

        if (local->ops->get_ftm_responder_stats)
                ret = local->ops->get_ftm_responder_stats(&local->hw,
                                                         &sdata->vif,
                                                         ftm_stats);
        trace_drv_get_ftm_responder_stats(local, sdata, ftm_stats);

        return ret;
}

static inline int drv_start_pmsr(struct ieee80211_local *local,
                                 struct ieee80211_sub_if_data *sdata,
                                 struct cfg80211_pmsr_request *request)
{
        int ret = -EOPNOTSUPP;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (!check_sdata_in_driver(sdata))
                return -EIO;

        trace_drv_start_pmsr(local, sdata);

        if (local->ops->start_pmsr)
                ret = local->ops->start_pmsr(&local->hw, &sdata->vif, request);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline void drv_abort_pmsr(struct ieee80211_local *local,
                                  struct ieee80211_sub_if_data *sdata,
                                  struct cfg80211_pmsr_request *request)
{
        trace_drv_abort_pmsr(local, sdata);

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (!check_sdata_in_driver(sdata))
                return;

        if (local->ops->abort_pmsr)
                local->ops->abort_pmsr(&local->hw, &sdata->vif, request);
        trace_drv_return_void(local);
}

static inline int drv_start_nan(struct ieee80211_local *local,
                                struct ieee80211_sub_if_data *sdata,
                                struct cfg80211_nan_conf *conf)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        check_sdata_in_driver(sdata);

        trace_drv_start_nan(local, sdata, conf);
        ret = local->ops->start_nan(&local->hw, &sdata->vif, conf);
        trace_drv_return_int(local, ret);
        return ret;
}

static inline void drv_stop_nan(struct ieee80211_local *local,
                                struct ieee80211_sub_if_data *sdata)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        check_sdata_in_driver(sdata);

        trace_drv_stop_nan(local, sdata);
        local->ops->stop_nan(&local->hw, &sdata->vif);
        trace_drv_return_void(local);
}

static inline int drv_nan_change_conf(struct ieee80211_local *local,
                                       struct ieee80211_sub_if_data *sdata,
                                       struct cfg80211_nan_conf *conf,
                                       u32 changes)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        check_sdata_in_driver(sdata);

        if (!local->ops->nan_change_conf)
                return -EOPNOTSUPP;

        trace_drv_nan_change_conf(local, sdata, conf, changes);
        ret = local->ops->nan_change_conf(&local->hw, &sdata->vif, conf,
                                          changes);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline int drv_add_nan_func(struct ieee80211_local *local,
                                   struct ieee80211_sub_if_data *sdata,
                                   const struct cfg80211_nan_func *nan_func)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        check_sdata_in_driver(sdata);

        if (!local->ops->add_nan_func)
                return -EOPNOTSUPP;

        trace_drv_add_nan_func(local, sdata, nan_func);
        ret = local->ops->add_nan_func(&local->hw, &sdata->vif, nan_func);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline void drv_del_nan_func(struct ieee80211_local *local,
                                   struct ieee80211_sub_if_data *sdata,
                                   u8 instance_id)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        check_sdata_in_driver(sdata);

        trace_drv_del_nan_func(local, sdata, instance_id);
        if (local->ops->del_nan_func)
                local->ops->del_nan_func(&local->hw, &sdata->vif, instance_id);
        trace_drv_return_void(local);
}

static inline int drv_set_tid_config(struct ieee80211_local *local,
                                     struct ieee80211_sub_if_data *sdata,
                                     struct ieee80211_sta *sta,
                                     struct cfg80211_tid_config *tid_conf)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        ret = local->ops->set_tid_config(&local->hw, &sdata->vif, sta,
                                         tid_conf);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline int drv_reset_tid_config(struct ieee80211_local *local,
                                       struct ieee80211_sub_if_data *sdata,
                                       struct ieee80211_sta *sta, u8 tids)
{
        int ret;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        ret = local->ops->reset_tid_config(&local->hw, &sdata->vif, sta, tids);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline void drv_update_vif_offload(struct ieee80211_local *local,
                                          struct ieee80211_sub_if_data *sdata)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        check_sdata_in_driver(sdata);

        if (!local->ops->update_vif_offload)
                return;

        trace_drv_update_vif_offload(local, sdata);
        local->ops->update_vif_offload(&local->hw, &sdata->vif);
        trace_drv_return_void(local);
}

static inline void drv_sta_set_4addr(struct ieee80211_local *local,
                                     struct ieee80211_sub_if_data *sdata,
                                     struct ieee80211_sta *sta, bool enabled)
{
        sdata = get_bss_sdata(sdata);

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_sta_set_4addr(local, sdata, sta, enabled);
        if (local->ops->sta_set_4addr)
                local->ops->sta_set_4addr(&local->hw, &sdata->vif, sta, enabled);
        trace_drv_return_void(local);
}

static inline void drv_sta_set_decap_offload(struct ieee80211_local *local,
                                             struct ieee80211_sub_if_data *sdata,
                                             struct ieee80211_sta *sta,
                                             bool enabled)
{
        sdata = get_bss_sdata(sdata);

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (!check_sdata_in_driver(sdata))
                return;

        trace_drv_sta_set_decap_offload(local, sdata, sta, enabled);
        if (local->ops->sta_set_decap_offload)
                local->ops->sta_set_decap_offload(&local->hw, &sdata->vif, sta,
                                                  enabled);
        trace_drv_return_void(local);
}

static inline void drv_add_twt_setup(struct ieee80211_local *local,
                                     struct ieee80211_sub_if_data *sdata,
                                     struct ieee80211_sta *sta,
                                     struct ieee80211_twt_setup *twt)
{
        struct ieee80211_twt_params *twt_agrt;

        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return;

        twt_agrt = (void *)twt->params;

        trace_drv_add_twt_setup(local, sta, twt, twt_agrt);
        local->ops->add_twt_setup(&local->hw, sta, twt);
        trace_drv_return_void(local);
}

static inline void drv_twt_teardown_request(struct ieee80211_local *local,
                                            struct ieee80211_sub_if_data *sdata,
                                            struct ieee80211_sta *sta,
                                            u8 flowid)
{
        might_sleep();
        lockdep_assert_wiphy(local->hw.wiphy);
        if (!check_sdata_in_driver(sdata))
                return;

        if (!local->ops->twt_teardown_request)
                return;

        trace_drv_twt_teardown_request(local, sta, flowid);
        local->ops->twt_teardown_request(&local->hw, sta, flowid);
        trace_drv_return_void(local);
}

static inline int drv_net_fill_forward_path(struct ieee80211_local *local,
                                            struct ieee80211_sub_if_data *sdata,
                                            struct ieee80211_sta *sta,
                                            struct net_device_path_ctx *ctx,
                                            struct net_device_path *path)
{
        int ret = -EOPNOTSUPP;

        sdata = get_bss_sdata(sdata);
        if (!check_sdata_in_driver(sdata))
                return -EIO;

        trace_drv_net_fill_forward_path(local, sdata, sta);
        if (local->ops->net_fill_forward_path)
                ret = local->ops->net_fill_forward_path(&local->hw,
                                                        &sdata->vif, sta,
                                                        ctx, path);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline int drv_net_setup_tc(struct ieee80211_local *local,
                                   struct ieee80211_sub_if_data *sdata,
                                   struct net_device *dev,
                                   enum tc_setup_type type, void *type_data)
{
        int ret = -EOPNOTSUPP;

        might_sleep();

        sdata = get_bss_sdata(sdata);
        trace_drv_net_setup_tc(local, sdata, type);
        if (local->ops->net_setup_tc)
                ret = local->ops->net_setup_tc(&local->hw, &sdata->vif, dev,
                                               type, type_data);
        trace_drv_return_int(local, ret);

        return ret;
}

static inline bool drv_can_activate_links(struct ieee80211_local *local,
                                          struct ieee80211_sub_if_data *sdata,
                                          u16 active_links)
{
        bool ret = true;

        lockdep_assert_wiphy(local->hw.wiphy);

        if (!check_sdata_in_driver(sdata))
                return false;

        trace_drv_can_activate_links(local, sdata, active_links);
        if (local->ops->can_activate_links)
                ret = local->ops->can_activate_links(&local->hw, &sdata->vif,
                                                     active_links);
        trace_drv_return_bool(local, ret);

        return ret;
}

int drv_change_vif_links(struct ieee80211_local *local,
                         struct ieee80211_sub_if_data *sdata,
                         u16 old_links, u16 new_links,
                         struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS]);
int drv_change_sta_links(struct ieee80211_local *local,
                         struct ieee80211_sub_if_data *sdata,
                         struct ieee80211_sta *sta,
                         u16 old_links, u16 new_links);

static inline enum ieee80211_neg_ttlm_res
drv_can_neg_ttlm(struct ieee80211_local *local,
                 struct ieee80211_sub_if_data *sdata,
                 struct ieee80211_neg_ttlm *neg_ttlm)
{
        enum ieee80211_neg_ttlm_res res = NEG_TTLM_RES_REJECT;

        might_sleep();
        if (!check_sdata_in_driver(sdata))
                return -EIO;

        trace_drv_can_neg_ttlm(local, sdata, neg_ttlm);
        if (local->ops->can_neg_ttlm)
                res = local->ops->can_neg_ttlm(&local->hw, &sdata->vif,
                                               neg_ttlm);
        trace_drv_neg_ttlm_res(local, sdata, res, neg_ttlm);

        return res;
}
#endif /* __MAC80211_DRIVER_OPS */




































































































































































































  132 




  132 

    4 







   28 



    6 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
/* SPDX-License-Identifier: GPL-2.0-or-later */

#ifndef _LINUX_RSTREASON_H
#define _LINUX_RSTREASON_H
#include <net/dropreason-core.h>
#include <uapi/linux/mptcp.h>

#define DEFINE_RST_REASON(FN, FNe)        \
        FN(NOT_SPECIFIED)                \
        FN(NO_SOCKET)                        \
        FN(TCP_INVALID_ACK_SEQUENCE)        \
        FN(TCP_RFC7323_PAWS)                \
        FN(TCP_TOO_OLD_ACK)                \
        FN(TCP_ACK_UNSENT_DATA)                \
        FN(TCP_FLAGS)                        \
        FN(TCP_OLD_ACK)                        \
        FN(TCP_ABORT_ON_DATA)                \
        FN(TCP_TIMEWAIT_SOCKET)                \
        FN(INVALID_SYN)                        \
        FN(TCP_ABORT_ON_CLOSE)                \
        FN(TCP_ABORT_ON_LINGER)                \
        FN(TCP_ABORT_ON_MEMORY)                \
        FN(TCP_STATE)                        \
        FN(TCP_KEEPALIVE_TIMEOUT)        \
        FN(TCP_DISCONNECT_WITH_DATA)        \
        FN(MPTCP_RST_EUNSPEC)                \
        FN(MPTCP_RST_EMPTCP)                \
        FN(MPTCP_RST_ERESOURCE)                \
        FN(MPTCP_RST_EPROHIBIT)                \
        FN(MPTCP_RST_EWQ2BIG)                \
        FN(MPTCP_RST_EBADPERF)                \
        FN(MPTCP_RST_EMIDDLEBOX)        \
        FN(ERROR)                        \
        FNe(MAX)

/**
 * enum sk_rst_reason - the reasons of socket reset
 *
 * The reasons of sk reset, which are used in DCCP/TCP/MPTCP protocols.
 *
 * There are three parts in order:
 * 1) skb drop reasons: relying on drop reasons for such as passive reset
 * 2) independent reset reasons: such as active reset reasons
 * 3) reset reasons in MPTCP: only for MPTCP use
 */
enum sk_rst_reason {
        /* Refer to include/net/dropreason-core.h
         * Rely on skb drop reasons because it indicates exactly why RST
         * could happen.
         */
        /** @SK_RST_REASON_NOT_SPECIFIED: reset reason is not specified */
        SK_RST_REASON_NOT_SPECIFIED,
        /** @SK_RST_REASON_NO_SOCKET: no valid socket that can be used */
        SK_RST_REASON_NO_SOCKET,
        /**
         * @SK_RST_REASON_TCP_INVALID_ACK_SEQUENCE: Not acceptable ACK SEQ
         * field because ack sequence is not in the window between snd_una
         * and snd_nxt
         */
        SK_RST_REASON_TCP_INVALID_ACK_SEQUENCE,
        /**
         * @SK_RST_REASON_TCP_RFC7323_PAWS: PAWS check, corresponding to
         * LINUX_MIB_PAWSESTABREJECTED, LINUX_MIB_PAWSACTIVEREJECTED
         */
        SK_RST_REASON_TCP_RFC7323_PAWS,
        /** @SK_RST_REASON_TCP_TOO_OLD_ACK: TCP ACK is too old */
        SK_RST_REASON_TCP_TOO_OLD_ACK,
        /**
         * @SK_RST_REASON_TCP_ACK_UNSENT_DATA: TCP ACK for data we haven't
         * sent yet
         */
        SK_RST_REASON_TCP_ACK_UNSENT_DATA,
        /** @SK_RST_REASON_TCP_FLAGS: TCP flags invalid */
        SK_RST_REASON_TCP_FLAGS,
        /** @SK_RST_REASON_TCP_OLD_ACK: TCP ACK is old, but in window */
        SK_RST_REASON_TCP_OLD_ACK,
        /**
         * @SK_RST_REASON_TCP_ABORT_ON_DATA: abort on data
         * corresponding to LINUX_MIB_TCPABORTONDATA
         */
        SK_RST_REASON_TCP_ABORT_ON_DATA,

        /* Here start with the independent reasons */
        /** @SK_RST_REASON_TCP_TIMEWAIT_SOCKET: happen on the timewait socket */
        SK_RST_REASON_TCP_TIMEWAIT_SOCKET,
        /**
         * @SK_RST_REASON_INVALID_SYN: receive bad syn packet
         * RFC 793 says if the state is not CLOSED/LISTEN/SYN-SENT then
         * "fourth, check the SYN bit,...If the SYN is in the window it is
         * an error, send a reset"
         */
        SK_RST_REASON_INVALID_SYN,
        /**
         * @SK_RST_REASON_TCP_ABORT_ON_CLOSE: abort on close
         * corresponding to LINUX_MIB_TCPABORTONCLOSE
         */
        SK_RST_REASON_TCP_ABORT_ON_CLOSE,
        /**
         * @SK_RST_REASON_TCP_ABORT_ON_LINGER: abort on linger
         * corresponding to LINUX_MIB_TCPABORTONLINGER
         */
        SK_RST_REASON_TCP_ABORT_ON_LINGER,
        /**
         * @SK_RST_REASON_TCP_ABORT_ON_MEMORY: abort on memory
         * corresponding to LINUX_MIB_TCPABORTONMEMORY
         */
        SK_RST_REASON_TCP_ABORT_ON_MEMORY,
        /**
         * @SK_RST_REASON_TCP_STATE: abort on tcp state
         * Please see RFC 9293 for all possible reset conditions
         */
        SK_RST_REASON_TCP_STATE,
        /**
         * @SK_RST_REASON_TCP_KEEPALIVE_TIMEOUT: time to timeout
         * When we have already run out of all the chances, which means
         * keepalive timeout, we have to reset the connection
         */
        SK_RST_REASON_TCP_KEEPALIVE_TIMEOUT,
        /**
         * @SK_RST_REASON_TCP_DISCONNECT_WITH_DATA: disconnect when write
         * queue is not empty
         * It means user has written data into the write queue when doing
         * disconnecting, so we have to send an RST.
         */
        SK_RST_REASON_TCP_DISCONNECT_WITH_DATA,

        /* Copy from include/uapi/linux/mptcp.h.
         * These reset fields will not be changed since they adhere to
         * RFC 8684. So do not touch them. I'm going to list each definition
         * of them respectively.
         */
        /**
         * @SK_RST_REASON_MPTCP_RST_EUNSPEC: Unspecified error.
         * This is the default error; it implies that the subflow is no
         * longer available. The presence of this option shows that the
         * RST was generated by an MPTCP-aware device.
         */
        SK_RST_REASON_MPTCP_RST_EUNSPEC,
        /**
         * @SK_RST_REASON_MPTCP_RST_EMPTCP: MPTCP-specific error.
         * An error has been detected in the processing of MPTCP options.
         * This is the usual reason code to return in the cases where a RST
         * is being sent to close a subflow because of an invalid response.
         */
        SK_RST_REASON_MPTCP_RST_EMPTCP,
        /**
         * @SK_RST_REASON_MPTCP_RST_ERESOURCE: Lack of resources.
         * This code indicates that the sending host does not have enough
         * resources to support the terminated subflow.
         */
        SK_RST_REASON_MPTCP_RST_ERESOURCE,
        /**
         * @SK_RST_REASON_MPTCP_RST_EPROHIBIT: Administratively prohibited.
         * This code indicates that the requested subflow is prohibited by
         * the policies of the sending host.
         */
        SK_RST_REASON_MPTCP_RST_EPROHIBIT,
        /**
         * @SK_RST_REASON_MPTCP_RST_EWQ2BIG: Too much outstanding data.
         * This code indicates that there is an excessive amount of data
         * that needs to be transmitted over the terminated subflow while
         * having already been acknowledged over one or more other subflows.
         * This may occur if a path has been unavailable for a short period
         * and it is more efficient to reset and start again than it is to
         * retransmit the queued data.
         */
        SK_RST_REASON_MPTCP_RST_EWQ2BIG,
        /**
         * @SK_RST_REASON_MPTCP_RST_EBADPERF: Unacceptable performance.
         * This code indicates that the performance of this subflow was
         * too low compared to the other subflows of this Multipath TCP
         * connection.
         */
        SK_RST_REASON_MPTCP_RST_EBADPERF,
        /**
         * @SK_RST_REASON_MPTCP_RST_EMIDDLEBOX: Middlebox interference.
         * Middlebox interference has been detected over this subflow,
         * making MPTCP signaling invalid. For example, this may be sent
         * if the checksum does not validate.
         */
        SK_RST_REASON_MPTCP_RST_EMIDDLEBOX,

        /** @SK_RST_REASON_ERROR: unexpected error happens */
        SK_RST_REASON_ERROR,

        /**
         * @SK_RST_REASON_MAX: Maximum of socket reset reasons.
         * It shouldn't be used as a real 'reason'.
         */
        SK_RST_REASON_MAX,
};

/* Convert skb drop reasons to enum sk_rst_reason type */
static inline enum sk_rst_reason
sk_rst_convert_drop_reason(enum skb_drop_reason reason)
{
        switch (reason) {
        case SKB_DROP_REASON_NOT_SPECIFIED:
                return SK_RST_REASON_NOT_SPECIFIED;
        case SKB_DROP_REASON_NO_SOCKET:
                return SK_RST_REASON_NO_SOCKET;
        case SKB_DROP_REASON_TCP_INVALID_ACK_SEQUENCE:
                return SK_RST_REASON_TCP_INVALID_ACK_SEQUENCE;
        case SKB_DROP_REASON_TCP_RFC7323_PAWS:
                return SK_RST_REASON_TCP_RFC7323_PAWS;
        case SKB_DROP_REASON_TCP_TOO_OLD_ACK:
                return SK_RST_REASON_TCP_TOO_OLD_ACK;
        case SKB_DROP_REASON_TCP_ACK_UNSENT_DATA:
                return SK_RST_REASON_TCP_ACK_UNSENT_DATA;
        case SKB_DROP_REASON_TCP_FLAGS:
                return SK_RST_REASON_TCP_FLAGS;
        case SKB_DROP_REASON_TCP_OLD_ACK:
                return SK_RST_REASON_TCP_OLD_ACK;
        case SKB_DROP_REASON_TCP_ABORT_ON_DATA:
                return SK_RST_REASON_TCP_ABORT_ON_DATA;
        default:
                /* If we don't have our own corresponding reason */
                return SK_RST_REASON_NOT_SPECIFIED;
        }
}
#endif





























































































   22 




    1 






    3 




   19 























   21 




















   15 
   10 



    8 

    2 
    6 

































































    9 

















    1 



    2 






    1 

    1 



    1 



    1 










    1 

















    1 
    1 




















































































































































































    1 

    1 



















































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
/*
 * Routines to compress and uncompress tcp packets (for transmission
 * over low speed serial lines).
 *
 * Copyright (c) 1989 Regents of the University of California.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms are permitted
 * provided that the above copyright notice and this paragraph are
 * duplicated in all such forms and that any documentation,
 * advertising materials, and other materials related to such
 * distribution and use acknowledge that the software was developed
 * by the University of California, Berkeley.  The name of the
 * University may not be used to endorse or promote products derived
 * from this software without specific prior written permission.
 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 *
 *        Van Jacobson (van@helios.ee.lbl.gov), Dec 31, 1989:
 *        - Initial distribution.
 *
 *
 * modified for KA9Q Internet Software Package by
 * Katie Stevens (dkstevens@ucdavis.edu)
 * University of California, Davis
 * Computing Services
 *        - 01-31-90        initial adaptation (from 1.19)
 *        PPP.05        02-15-90 [ks]
 *        PPP.08        05-02-90 [ks]        use PPP protocol field to signal compression
 *        PPP.15        09-90         [ks]        improve mbuf handling
 *        PPP.16        11-02         [karn]        substantially rewritten to use NOS facilities
 *
 *        - Feb 1991        Bill_Simpson@um.cc.umich.edu
 *                        variable number of conversation slots
 *                        allow zero or one slots
 *                        separate routines
 *                        status display
 *        - Jul 1994        Dmitry Gorodchanin
 *                        Fixes for memory leaks.
 *      - Oct 1994      Dmitry Gorodchanin
 *                      Modularization.
 *        - Jan 1995        Bjorn Ekwall
 *                        Use ip_fast_csum from ip.h
 *        - July 1995        Christos A. Polyzols
 *                        Spotted bug in tcp option checking
 *
 *
 *        This module is a difficult issue. It's clearly inet code but it's also clearly
 *        driver code belonging close to PPP and SLIP
 */

#include <linux/module.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/kernel.h>
#include <net/slhc_vj.h>

#ifdef CONFIG_INET
/* Entire module is for IP only */
#include <linux/mm.h>
#include <linux/socket.h>
#include <linux/sockios.h>
#include <linux/termios.h>
#include <linux/in.h>
#include <linux/fcntl.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <net/ip.h>
#include <net/protocol.h>
#include <net/icmp.h>
#include <net/tcp.h>
#include <linux/skbuff.h>
#include <net/sock.h>
#include <linux/timer.h>
#include <linux/uaccess.h>
#include <net/checksum.h>
#include <asm/unaligned.h>

static unsigned char *encode(unsigned char *cp, unsigned short n);
static long decode(unsigned char **cpp);
static unsigned char * put16(unsigned char *cp, unsigned short x);
static unsigned short pull16(unsigned char **cpp);

/* Allocate compression data structure
 *        slots must be in range 0 to 255 (zero meaning no compression)
 * Returns pointer to structure or ERR_PTR() on error.
 */
struct slcompress *
slhc_init(int rslots, int tslots)
{
        short i;
        struct cstate *ts;
        struct slcompress *comp;

        if (rslots < 0 || rslots > 255 || tslots < 0 || tslots > 255)
                return ERR_PTR(-EINVAL);

        comp = kzalloc(sizeof(struct slcompress), GFP_KERNEL);
        if (! comp)
                goto out_fail;

        if (rslots > 0) {
                size_t rsize = rslots * sizeof(struct cstate);
                comp->rstate = kzalloc(rsize, GFP_KERNEL);
                if (! comp->rstate)
                        goto out_free;
                comp->rslot_limit = rslots - 1;
        }

        if (tslots > 0) {
                size_t tsize = tslots * sizeof(struct cstate);
                comp->tstate = kzalloc(tsize, GFP_KERNEL);
                if (! comp->tstate)
                        goto out_free2;
                comp->tslot_limit = tslots - 1;
        }

        comp->xmit_oldest = 0;
        comp->xmit_current = 255;
        comp->recv_current = 255;
        /*
         * don't accept any packets with implicit index until we get
         * one with an explicit index.  Otherwise the uncompress code
         * will try to use connection 255, which is almost certainly
         * out of range
         */
        comp->flags |= SLF_TOSS;

        if ( tslots > 0 ) {
                ts = comp->tstate;
                for(i = comp->tslot_limit; i > 0; --i){
                        ts[i].cs_this = i;
                        ts[i].next = &(ts[i - 1]);
                }
                ts[0].next = &(ts[comp->tslot_limit]);
                ts[0].cs_this = 0;
        }
        return comp;

out_free2:
        kfree(comp->rstate);
out_free:
        kfree(comp);
out_fail:
        return ERR_PTR(-ENOMEM);
}


/* Free a compression data structure */
void
slhc_free(struct slcompress *comp)
{
        if ( IS_ERR_OR_NULL(comp) )
                return;

        if ( comp->tstate != NULLSLSTATE )
                kfree( comp->tstate );

        if ( comp->rstate != NULLSLSTATE )
                kfree( comp->rstate );

        kfree( comp );
}


/* Put a short in host order into a char array in network order */
static inline unsigned char *
put16(unsigned char *cp, unsigned short x)
{
        *cp++ = x >> 8;
        *cp++ = x;

        return cp;
}


/* Encode a number */
static unsigned char *
encode(unsigned char *cp, unsigned short n)
{
        if(n >= 256 || n == 0){
                *cp++ = 0;
                cp = put16(cp,n);
        } else {
                *cp++ = n;
        }
        return cp;
}

/* Pull a 16-bit integer in host order from buffer in network byte order */
static unsigned short
pull16(unsigned char **cpp)
{
        short rval;

        rval = *(*cpp)++;
        rval <<= 8;
        rval |= *(*cpp)++;
        return rval;
}

/* Decode a number */
static long
decode(unsigned char **cpp)
{
        int x;

        x = *(*cpp)++;
        if(x == 0){
                return pull16(cpp) & 0xffff;        /* pull16 returns -1 on error */
        } else {
                return x & 0xff;                /* -1 if PULLCHAR returned error */
        }
}

/*
 * icp and isize are the original packet.
 * ocp is a place to put a copy if necessary.
 * cpp is initially a pointer to icp.  If the copy is used,
 *    change it to ocp.
 */

int
slhc_compress(struct slcompress *comp, unsigned char *icp, int isize,
        unsigned char *ocp, unsigned char **cpp, int compress_cid)
{
        struct cstate *ocs = &(comp->tstate[comp->xmit_oldest]);
        struct cstate *lcs = ocs;
        struct cstate *cs = lcs->next;
        unsigned long deltaS, deltaA;
        short changes = 0;
        int nlen, hlen;
        unsigned char new_seq[16];
        unsigned char *cp = new_seq;
        struct iphdr *ip;
        struct tcphdr *th, *oth;
        __sum16 csum;


        /*
         *        Don't play with runt packets.
         */

        if(isize<sizeof(struct iphdr))
                return isize;

        ip = (struct iphdr *) icp;
        if (ip->version != 4 || ip->ihl < 5)
                return isize;

        /* Bail if this packet isn't TCP, or is an IP fragment */
        if (ip->protocol != IPPROTO_TCP || (ntohs(ip->frag_off) & 0x3fff)) {
                /* Send as regular IP */
                if(ip->protocol != IPPROTO_TCP)
                        comp->sls_o_nontcp++;
                else
                        comp->sls_o_tcp++;
                return isize;
        }
        nlen = ip->ihl * 4;
        if (isize < nlen + sizeof(*th))
                return isize;

        th = (struct tcphdr *)(icp + nlen);
        if (th->doff < sizeof(struct tcphdr) / 4)
                return isize;
        hlen = nlen + th->doff * 4;

        /*  Bail if the TCP packet isn't `compressible' (i.e., ACK isn't set or
         *  some other control bit is set). Also uncompressible if
         *  it's a runt.
         */
        if(hlen > isize || th->syn || th->fin || th->rst ||
            ! (th->ack)){
                /* TCP connection stuff; send as regular IP */
                comp->sls_o_tcp++;
                return isize;
        }
        /*
         * Packet is compressible -- we're going to send either a
         * COMPRESSED_TCP or UNCOMPRESSED_TCP packet.  Either way,
         * we need to locate (or create) the connection state.
         *
         * States are kept in a circularly linked list with
         * xmit_oldest pointing to the end of the list.  The
         * list is kept in lru order by moving a state to the
         * head of the list whenever it is referenced.  Since
         * the list is short and, empirically, the connection
         * we want is almost always near the front, we locate
         * states via linear search.  If we don't find a state
         * for the datagram, the oldest state is (re-)used.
         */
        for ( ; ; ) {
                if( ip->saddr == cs->cs_ip.saddr
                 && ip->daddr == cs->cs_ip.daddr
                 && th->source == cs->cs_tcp.source
                 && th->dest == cs->cs_tcp.dest)
                        goto found;

                /* if current equal oldest, at end of list */
                if ( cs == ocs )
                        break;
                lcs = cs;
                cs = cs->next;
                comp->sls_o_searches++;
        }
        /*
         * Didn't find it -- re-use oldest cstate.  Send an
         * uncompressed packet that tells the other side what
         * connection number we're using for this conversation.
         *
         * Note that since the state list is circular, the oldest
         * state points to the newest and we only need to set
         * xmit_oldest to update the lru linkage.
         */
        comp->sls_o_misses++;
        comp->xmit_oldest = lcs->cs_this;
        goto uncompressed;

found:
        /*
         * Found it -- move to the front on the connection list.
         */
        if(lcs == ocs) {
                /* found at most recently used */
        } else if (cs == ocs) {
                /* found at least recently used */
                comp->xmit_oldest = lcs->cs_this;
        } else {
                /* more than 2 elements */
                lcs->next = cs->next;
                cs->next = ocs->next;
                ocs->next = cs;
        }

        /*
         * Make sure that only what we expect to change changed.
         * Check the following:
         * IP protocol version, header length & type of service.
         * The "Don't fragment" bit.
         * The time-to-live field.
         * The TCP header length.
         * IP options, if any.
         * TCP options, if any.
         * If any of these things are different between the previous &
         * current datagram, we send the current datagram `uncompressed'.
         */
        oth = &cs->cs_tcp;

        if(ip->version != cs->cs_ip.version || ip->ihl != cs->cs_ip.ihl
         || ip->tos != cs->cs_ip.tos
         || (ip->frag_off & htons(0x4000)) != (cs->cs_ip.frag_off & htons(0x4000))
         || ip->ttl != cs->cs_ip.ttl
         || th->doff != cs->cs_tcp.doff
         || (ip->ihl > 5 && memcmp(ip+1,cs->cs_ipopt,((ip->ihl)-5)*4) != 0)
         || (th->doff > 5 && memcmp(th+1,cs->cs_tcpopt,((th->doff)-5)*4) != 0)){
                goto uncompressed;
        }

        /*
         * Figure out which of the changing fields changed.  The
         * receiver expects changes in the order: urgent, window,
         * ack, seq (the order minimizes the number of temporaries
         * needed in this section of code).
         */
        if(th->urg){
                deltaS = ntohs(th->urg_ptr);
                cp = encode(cp,deltaS);
                changes |= NEW_U;
        } else if(th->urg_ptr != oth->urg_ptr){
                /* argh! URG not set but urp changed -- a sensible
                 * implementation should never do this but RFC793
                 * doesn't prohibit the change so we have to deal
                 * with it. */
                goto uncompressed;
        }
        if((deltaS = ntohs(th->window) - ntohs(oth->window)) != 0){
                cp = encode(cp,deltaS);
                changes |= NEW_W;
        }
        if((deltaA = ntohl(th->ack_seq) - ntohl(oth->ack_seq)) != 0L){
                if(deltaA > 0x0000ffff)
                        goto uncompressed;
                cp = encode(cp,deltaA);
                changes |= NEW_A;
        }
        if((deltaS = ntohl(th->seq) - ntohl(oth->seq)) != 0L){
                if(deltaS > 0x0000ffff)
                        goto uncompressed;
                cp = encode(cp,deltaS);
                changes |= NEW_S;
        }

        switch(changes){
        case 0:        /* Nothing changed. If this packet contains data and the
                 * last one didn't, this is probably a data packet following
                 * an ack (normal on an interactive connection) and we send
                 * it compressed.  Otherwise it's probably a retransmit,
                 * retransmitted ack or window probe.  Send it uncompressed
                 * in case the other side missed the compressed version.
                 */
                if(ip->tot_len != cs->cs_ip.tot_len &&
                   ntohs(cs->cs_ip.tot_len) == hlen)
                        break;
                goto uncompressed;
        case SPECIAL_I:
        case SPECIAL_D:
                /* actual changes match one of our special case encodings --
                 * send packet uncompressed.
                 */
                goto uncompressed;
        case NEW_S|NEW_A:
                if(deltaS == deltaA &&
                    deltaS == ntohs(cs->cs_ip.tot_len) - hlen){
                        /* special case for echoed terminal traffic */
                        changes = SPECIAL_I;
                        cp = new_seq;
                }
                break;
        case NEW_S:
                if(deltaS == ntohs(cs->cs_ip.tot_len) - hlen){
                        /* special case for data xfer */
                        changes = SPECIAL_D;
                        cp = new_seq;
                }
                break;
        }
        deltaS = ntohs(ip->id) - ntohs(cs->cs_ip.id);
        if(deltaS != 1){
                cp = encode(cp,deltaS);
                changes |= NEW_I;
        }
        if(th->psh)
                changes |= TCP_PUSH_BIT;
        /* Grab the cksum before we overwrite it below.  Then update our
         * state with this packet's header.
         */
        csum = th->check;
        memcpy(&cs->cs_ip,ip,20);
        memcpy(&cs->cs_tcp,th,20);
        /* We want to use the original packet as our compressed packet.
         * (cp - new_seq) is the number of bytes we need for compressed
         * sequence numbers.  In addition we need one byte for the change
         * mask, one for the connection id and two for the tcp checksum.
         * So, (cp - new_seq) + 4 bytes of header are needed.
         */
        deltaS = cp - new_seq;
        if(compress_cid == 0 || comp->xmit_current != cs->cs_this){
                cp = ocp;
                *cpp = ocp;
                *cp++ = changes | NEW_C;
                *cp++ = cs->cs_this;
                comp->xmit_current = cs->cs_this;
        } else {
                cp = ocp;
                *cpp = ocp;
                *cp++ = changes;
        }
        *(__sum16 *)cp = csum;
        cp += 2;
/* deltaS is now the size of the change section of the compressed header */
        memcpy(cp,new_seq,deltaS);        /* Write list of deltas */
        memcpy(cp+deltaS,icp+hlen,isize-hlen);
        comp->sls_o_compressed++;
        ocp[0] |= SL_TYPE_COMPRESSED_TCP;
        return isize - hlen + deltaS + (cp - ocp);

        /* Update connection state cs & send uncompressed packet (i.e.,
         * a regular ip/tcp packet but with the 'conversation id' we hope
         * to use on future compressed packets in the protocol field).
         */
uncompressed:
        memcpy(&cs->cs_ip,ip,20);
        memcpy(&cs->cs_tcp,th,20);
        if (ip->ihl > 5)
          memcpy(cs->cs_ipopt, ip+1, ((ip->ihl) - 5) * 4);
        if (th->doff > 5)
          memcpy(cs->cs_tcpopt, th+1, ((th->doff) - 5) * 4);
        comp->xmit_current = cs->cs_this;
        comp->sls_o_uncompressed++;
        memcpy(ocp, icp, isize);
        *cpp = ocp;
        ocp[9] = cs->cs_this;
        ocp[0] |= SL_TYPE_UNCOMPRESSED_TCP;
        return isize;
}


int
slhc_uncompress(struct slcompress *comp, unsigned char *icp, int isize)
{
        int changes;
        long x;
        struct tcphdr *thp;
        struct iphdr *ip;
        struct cstate *cs;
        int len, hdrlen;
        unsigned char *cp = icp;

        /* We've got a compressed packet; read the change byte */
        comp->sls_i_compressed++;
        if(isize < 3){
                comp->sls_i_error++;
                return 0;
        }
        changes = *cp++;
        if(changes & NEW_C){
                /* Make sure the state index is in range, then grab the state.
                 * If we have a good state index, clear the 'discard' flag.
                 */
                x = *cp++;        /* Read conn index */
                if(x < 0 || x > comp->rslot_limit)
                        goto bad;

                /* Check if the cstate is initialized */
                if (!comp->rstate[x].initialized)
                        goto bad;

                comp->flags &=~ SLF_TOSS;
                comp->recv_current = x;
        } else {
                /* this packet has an implicit state index.  If we've
                 * had a line error since the last time we got an
                 * explicit state index, we have to toss the packet. */
                if(comp->flags & SLF_TOSS){
                        comp->sls_i_tossed++;
                        return 0;
                }
        }
        cs = &comp->rstate[comp->recv_current];
        thp = &cs->cs_tcp;
        ip = &cs->cs_ip;

        thp->check = *(__sum16 *)cp;
        cp += 2;

        thp->psh = (changes & TCP_PUSH_BIT) ? 1 : 0;
/*
 * we can use the same number for the length of the saved header and
 * the current one, because the packet wouldn't have been sent
 * as compressed unless the options were the same as the previous one
 */

        hdrlen = ip->ihl * 4 + thp->doff * 4;

        switch(changes & SPECIALS_MASK){
        case SPECIAL_I:                /* Echoed terminal traffic */
                {
                short i;
                i = ntohs(ip->tot_len) - hdrlen;
                thp->ack_seq = htonl( ntohl(thp->ack_seq) + i);
                thp->seq = htonl( ntohl(thp->seq) + i);
                }
                break;

        case SPECIAL_D:                        /* Unidirectional data */
                thp->seq = htonl( ntohl(thp->seq) +
                                  ntohs(ip->tot_len) - hdrlen);
                break;

        default:
                if(changes & NEW_U){
                        thp->urg = 1;
                        if((x = decode(&cp)) == -1) {
                                goto bad;
                        }
                        thp->urg_ptr = htons(x);
                } else
                        thp->urg = 0;
                if(changes & NEW_W){
                        if((x = decode(&cp)) == -1) {
                                goto bad;
                        }
                        thp->window = htons( ntohs(thp->window) + x);
                }
                if(changes & NEW_A){
                        if((x = decode(&cp)) == -1) {
                                goto bad;
                        }
                        thp->ack_seq = htonl( ntohl(thp->ack_seq) + x);
                }
                if(changes & NEW_S){
                        if((x = decode(&cp)) == -1) {
                                goto bad;
                        }
                        thp->seq = htonl( ntohl(thp->seq) + x);
                }
                break;
        }
        if(changes & NEW_I){
                if((x = decode(&cp)) == -1) {
                        goto bad;
                }
                ip->id = htons (ntohs (ip->id) + x);
        } else
                ip->id = htons (ntohs (ip->id) + 1);

        /*
         * At this point, cp points to the first byte of data in the
         * packet.  Put the reconstructed TCP and IP headers back on the
         * packet.  Recalculate IP checksum (but not TCP checksum).
         */

        len = isize - (cp - icp);
        if (len < 0)
                goto bad;
        len += hdrlen;
        ip->tot_len = htons(len);
        ip->check = 0;

        memmove(icp + hdrlen, cp, len - hdrlen);

        cp = icp;
        memcpy(cp, ip, 20);
        cp += 20;

        if (ip->ihl > 5) {
          memcpy(cp, cs->cs_ipopt, (ip->ihl - 5) * 4);
          cp += (ip->ihl - 5) * 4;
        }

        put_unaligned(ip_fast_csum(icp, ip->ihl),
                      &((struct iphdr *)icp)->check);

        memcpy(cp, thp, 20);
        cp += 20;

        if (thp->doff > 5) {
          memcpy(cp, cs->cs_tcpopt, ((thp->doff) - 5) * 4);
          cp += ((thp->doff) - 5) * 4;
        }

        return len;
bad:
        comp->sls_i_error++;
        return slhc_toss( comp );
}


int
slhc_remember(struct slcompress *comp, unsigned char *icp, int isize)
{
        struct cstate *cs;
        unsigned ihl;

        unsigned char index;

        if(isize < 20) {
                /* The packet is shorter than a legal IP header */
                comp->sls_i_runt++;
                return slhc_toss( comp );
        }
        /* Peek at the IP header's IHL field to find its length */
        ihl = icp[0] & 0xf;
        if(ihl < 20 / 4){
                /* The IP header length field is too small */
                comp->sls_i_runt++;
                return slhc_toss( comp );
        }
        index = icp[9];
        icp[9] = IPPROTO_TCP;

        if (ip_fast_csum(icp, ihl)) {
                /* Bad IP header checksum; discard */
                comp->sls_i_badcheck++;
                return slhc_toss( comp );
        }
        if(index > comp->rslot_limit) {
                comp->sls_i_error++;
                return slhc_toss(comp);
        }

        /* Update local state */
        cs = &comp->rstate[comp->recv_current = index];
        comp->flags &=~ SLF_TOSS;
        memcpy(&cs->cs_ip,icp,20);
        memcpy(&cs->cs_tcp,icp + ihl*4,20);
        if (ihl > 5)
          memcpy(cs->cs_ipopt, icp + sizeof(struct iphdr), (ihl - 5) * 4);
        if (cs->cs_tcp.doff > 5)
          memcpy(cs->cs_tcpopt, icp + ihl*4 + sizeof(struct tcphdr), (cs->cs_tcp.doff - 5) * 4);
        cs->cs_hsize = ihl*2 + cs->cs_tcp.doff*2;
        cs->initialized = true;
        /* Put headers back on packet
         * Neither header checksum is recalculated
         */
        comp->sls_i_uncompressed++;
        return isize;
}

int
slhc_toss(struct slcompress *comp)
{
        if ( comp == NULLSLCOMPR )
                return 0;

        comp->flags |= SLF_TOSS;
        return 0;
}

#else /* CONFIG_INET */

int
slhc_toss(struct slcompress *comp)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_toss");
  return -EINVAL;
}
int
slhc_uncompress(struct slcompress *comp, unsigned char *icp, int isize)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_uncompress");
  return -EINVAL;
}
int
slhc_compress(struct slcompress *comp, unsigned char *icp, int isize,
        unsigned char *ocp, unsigned char **cpp, int compress_cid)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_compress");
  return -EINVAL;
}

int
slhc_remember(struct slcompress *comp, unsigned char *icp, int isize)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_remember");
  return -EINVAL;
}

void
slhc_free(struct slcompress *comp)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_free");
}
struct slcompress *
slhc_init(int rslots, int tslots)
{
  printk(KERN_DEBUG "Called IP function on non IP-system: slhc_init");
  return NULL;
}

#endif /* CONFIG_INET */

/* VJ header compression */
EXPORT_SYMBOL(slhc_init);
EXPORT_SYMBOL(slhc_free);
EXPORT_SYMBOL(slhc_remember);
EXPORT_SYMBOL(slhc_compress);
EXPORT_SYMBOL(slhc_uncompress);
EXPORT_SYMBOL(slhc_toss);

MODULE_DESCRIPTION("Compression helpers for SLIP (serial line)");
MODULE_LICENSE("Dual BSD/GPL");






















   13 


   13 

    6 




   12 









































   19 












   19 

   19 











1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2015, Heiner Kallweit <hkallweit1@gmail.com>
 */

#include <net/bluetooth/bluetooth.h>
#include <net/bluetooth/hci_core.h>

#include "leds.h"

DEFINE_LED_TRIGGER(bt_power_led_trigger);

struct hci_basic_led_trigger {
        struct led_trigger        led_trigger;
        struct hci_dev                *hdev;
};

#define to_hci_basic_led_trigger(arg) container_of(arg, \
                        struct hci_basic_led_trigger, led_trigger)

void hci_leds_update_powered(struct hci_dev *hdev, bool enabled)
{
        if (hdev->power_led)
                led_trigger_event(hdev->power_led,
                                  enabled ? LED_FULL : LED_OFF);

        if (!enabled) {
                struct hci_dev *d;

                read_lock(&hci_dev_list_lock);

                list_for_each_entry(d, &hci_dev_list, list) {
                        if (test_bit(HCI_UP, &d->flags))
                                enabled = true;
                }

                read_unlock(&hci_dev_list_lock);
        }

        led_trigger_event(bt_power_led_trigger, enabled ? LED_FULL : LED_OFF);
}

static int power_activate(struct led_classdev *led_cdev)
{
        struct hci_basic_led_trigger *htrig;
        bool powered;

        htrig = to_hci_basic_led_trigger(led_cdev->trigger);
        powered = test_bit(HCI_UP, &htrig->hdev->flags);

        led_set_brightness(led_cdev, powered ? LED_FULL : LED_OFF);

        return 0;
}

static struct led_trigger *led_allocate_basic(struct hci_dev *hdev,
                        int (*activate)(struct led_classdev *led_cdev),
                        const char *name)
{
        struct hci_basic_led_trigger *htrig;

        htrig =        devm_kzalloc(&hdev->dev, sizeof(*htrig), GFP_KERNEL);
        if (!htrig)
                return NULL;

        htrig->hdev = hdev;
        htrig->led_trigger.activate = activate;
        htrig->led_trigger.name = devm_kasprintf(&hdev->dev, GFP_KERNEL,
                                                 "%s-%s", hdev->name,
                                                 name);
        if (!htrig->led_trigger.name)
                goto err_alloc;

        if (devm_led_trigger_register(&hdev->dev, &htrig->led_trigger))
                goto err_register;

        return &htrig->led_trigger;

err_register:
        devm_kfree(&hdev->dev, (void *)htrig->led_trigger.name);
err_alloc:
        devm_kfree(&hdev->dev, htrig);
        return NULL;
}

void hci_leds_init(struct hci_dev *hdev)
{
        /* initialize power_led */
        hdev->power_led = led_allocate_basic(hdev, power_activate, "power");
}

void bt_leds_init(void)
{
        led_trigger_register_simple("bluetooth-power", &bt_power_led_trigger);
}

void bt_leds_cleanup(void)
{
        led_trigger_unregister_simple(bt_power_led_trigger);
}





































  201 

   95 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef __LINUX_COOKIE_H
#define __LINUX_COOKIE_H

#include <linux/atomic.h>
#include <linux/percpu.h>
#include <asm/local.h>

struct pcpu_gen_cookie {
        local_t nesting;
        u64 last;
} __aligned(16);

struct gen_cookie {
        struct pcpu_gen_cookie __percpu *local;
        atomic64_t forward_last ____cacheline_aligned_in_smp;
        atomic64_t reverse_last;
};

#define COOKIE_LOCAL_BATCH        4096

#define DEFINE_COOKIE(name)                                                \
        static DEFINE_PER_CPU(struct pcpu_gen_cookie, __##name);        \
        static struct gen_cookie name = {                                \
                .local                = &__##name,                                \
                .forward_last        = ATOMIC64_INIT(0),                        \
                .reverse_last        = ATOMIC64_INIT(0),                        \
        }

static __always_inline u64 gen_cookie_next(struct gen_cookie *gc)
{
        struct pcpu_gen_cookie *local = this_cpu_ptr(gc->local);
        u64 val;

        if (likely(local_inc_return(&local->nesting) == 1)) {
                val = local->last;
                if (__is_defined(CONFIG_SMP) &&
                    unlikely((val & (COOKIE_LOCAL_BATCH - 1)) == 0)) {
                        s64 next = atomic64_add_return(COOKIE_LOCAL_BATCH,
                                                       &gc->forward_last);
                        val = next - COOKIE_LOCAL_BATCH;
                }
                local->last = ++val;
        } else {
                val = atomic64_dec_return(&gc->reverse_last);
        }
        local_dec(&local->nesting);
        return val;
}

#endif /* __LINUX_COOKIE_H */


























 1078 









  316 














































































































































































































































































































































































































































































































































  118 

































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * INET                An implementation of the TCP/IP protocol suite for the LINUX
 *                operating system.  INET is implemented using the  BSD Socket
 *                interface as the means of communication with the user level.
 *
 *                Definitions for the TCP protocol.
 *
 * Version:        @(#)tcp.h        1.0.2        04/28/93
 *
 * Author:        Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
 */
#ifndef _LINUX_TCP_H
#define _LINUX_TCP_H


#include <linux/skbuff.h>
#include <linux/win_minmax.h>
#include <net/sock.h>
#include <net/inet_connection_sock.h>
#include <net/inet_timewait_sock.h>
#include <uapi/linux/tcp.h>

static inline struct tcphdr *tcp_hdr(const struct sk_buff *skb)
{
        return (struct tcphdr *)skb_transport_header(skb);
}

static inline unsigned int __tcp_hdrlen(const struct tcphdr *th)
{
        return th->doff * 4;
}

static inline unsigned int tcp_hdrlen(const struct sk_buff *skb)
{
        return __tcp_hdrlen(tcp_hdr(skb));
}

static inline struct tcphdr *inner_tcp_hdr(const struct sk_buff *skb)
{
        return (struct tcphdr *)skb_inner_transport_header(skb);
}

static inline unsigned int inner_tcp_hdrlen(const struct sk_buff *skb)
{
        return inner_tcp_hdr(skb)->doff * 4;
}

/**
 * skb_tcp_all_headers - Returns size of all headers for a TCP packet
 * @skb: buffer
 *
 * Used in TX path, for a packet known to be a TCP one.
 *
 * if (skb_is_gso(skb)) {
 *         int hlen = skb_tcp_all_headers(skb);
 *         ...
 */
static inline int skb_tcp_all_headers(const struct sk_buff *skb)
{
        return skb_transport_offset(skb) + tcp_hdrlen(skb);
}

/**
 * skb_inner_tcp_all_headers - Returns size of all headers for an encap TCP packet
 * @skb: buffer
 *
 * Used in TX path, for a packet known to be a TCP one.
 *
 * if (skb_is_gso(skb) && skb->encapsulation) {
 *         int hlen = skb_inner_tcp_all_headers(skb);
 *         ...
 */
static inline int skb_inner_tcp_all_headers(const struct sk_buff *skb)
{
        return skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb);
}

static inline unsigned int tcp_optlen(const struct sk_buff *skb)
{
        return (tcp_hdr(skb)->doff - 5) * 4;
}

/* TCP Fast Open */
#define TCP_FASTOPEN_COOKIE_MIN        4        /* Min Fast Open Cookie size in bytes */
#define TCP_FASTOPEN_COOKIE_MAX        16        /* Max Fast Open Cookie size in bytes */
#define TCP_FASTOPEN_COOKIE_SIZE 8        /* the size employed by this impl. */

/* TCP Fast Open Cookie as stored in memory */
struct tcp_fastopen_cookie {
        __le64        val[DIV_ROUND_UP(TCP_FASTOPEN_COOKIE_MAX, sizeof(u64))];
        s8        len;
        bool        exp;        /* In RFC6994 experimental option format */
};

/* This defines a selective acknowledgement block. */
struct tcp_sack_block_wire {
        __be32        start_seq;
        __be32        end_seq;
};

struct tcp_sack_block {
        u32        start_seq;
        u32        end_seq;
};

/*These are used to set the sack_ok field in struct tcp_options_received */
#define TCP_SACK_SEEN     (1 << 0)   /*1 = peer is SACK capable, */
#define TCP_DSACK_SEEN    (1 << 2)   /*1 = DSACK was received from peer*/

struct tcp_options_received {
/*        PAWS/RTTM data        */
        int        ts_recent_stamp;/* Time we stored ts_recent (for aging) */
        u32        ts_recent;        /* Time stamp to echo next                */
        u32        rcv_tsval;        /* Time stamp value                     */
        u32        rcv_tsecr;        /* Time stamp echo reply                */
        u16         saw_tstamp : 1,        /* Saw TIMESTAMP on last packet                */
                tstamp_ok : 1,        /* TIMESTAMP seen on SYN packet                */
                dsack : 1,        /* D-SACK is scheduled                        */
                wscale_ok : 1,        /* Wscale seen on SYN packet                */
                sack_ok : 3,        /* SACK seen on SYN packet                */
                smc_ok : 1,        /* SMC seen on SYN packet                */
                snd_wscale : 4,        /* Window scaling received from sender        */
                rcv_wscale : 4;        /* Window scaling to send to receiver        */
        u8        saw_unknown:1,        /* Received unknown option                */
                unused:7;
        u8        num_sacks;        /* Number of SACK blocks                */
        u16        user_mss;        /* mss requested by user in ioctl        */
        u16        mss_clamp;        /* Maximal mss, negotiated at connection setup */
};

static inline void tcp_clear_options(struct tcp_options_received *rx_opt)
{
        rx_opt->tstamp_ok = rx_opt->sack_ok = 0;
        rx_opt->wscale_ok = rx_opt->snd_wscale = 0;
#if IS_ENABLED(CONFIG_SMC)
        rx_opt->smc_ok = 0;
#endif
}

/* This is the max number of SACKS that we'll generate and process. It's safe
 * to increase this, although since:
 *   size = TCPOLEN_SACK_BASE_ALIGNED (4) + n * TCPOLEN_SACK_PERBLOCK (8)
 * only four options will fit in a standard TCP header */
#define TCP_NUM_SACKS 4

struct tcp_request_sock_ops;

struct tcp_request_sock {
        struct inet_request_sock         req;
        const struct tcp_request_sock_ops *af_specific;
        u64                                snt_synack; /* first SYNACK sent time */
        bool                                tfo_listener;
        bool                                is_mptcp;
        bool                                req_usec_ts;
#if IS_ENABLED(CONFIG_MPTCP)
        bool                                drop_req;
#endif
        u32                                txhash;
        u32                                rcv_isn;
        u32                                snt_isn;
        u32                                ts_off;
        u32                                last_oow_ack_time; /* last SYNACK */
        u32                                rcv_nxt; /* the ack # by SYNACK. For
                                                  * FastOpen it's the seq#
                                                  * after data-in-SYN.
                                                  */
        u8                                syn_tos;
#ifdef CONFIG_TCP_AO
        u8                                ao_keyid;
        u8                                ao_rcv_next;
        bool                                used_tcp_ao;
#endif
};

static inline struct tcp_request_sock *tcp_rsk(const struct request_sock *req)
{
        return (struct tcp_request_sock *)req;
}

static inline bool tcp_rsk_used_ao(const struct request_sock *req)
{
#ifndef CONFIG_TCP_AO
        return false;
#else
        return tcp_rsk(req)->used_tcp_ao;
#endif
}

#define TCP_RMEM_TO_WIN_SCALE 8

struct tcp_sock {
        /* Cacheline organization can be found documented in
         * Documentation/networking/net_cachelines/tcp_sock.rst.
         * Please update the document when adding new fields.
         */

        /* inet_connection_sock has to be the first member of tcp_sock */
        struct inet_connection_sock        inet_conn;

        /* TX read-mostly hotpath cache lines */
        __cacheline_group_begin(tcp_sock_read_tx);
        /* timestamp of last sent data packet (for restart window) */
        u32        max_window;        /* Maximal window ever seen from peer        */
        u32        rcv_ssthresh;        /* Current window clamp                        */
        u32        reordering;        /* Packet reordering metric.                */
        u32        notsent_lowat;        /* TCP_NOTSENT_LOWAT */
        u16        gso_segs;        /* Max number of segs per GSO packet        */
        /* from STCP, retrans queue hinting */
        struct sk_buff *lost_skb_hint;
        struct sk_buff *retransmit_skb_hint;
        __cacheline_group_end(tcp_sock_read_tx);

        /* TXRX read-mostly hotpath cache lines */
        __cacheline_group_begin(tcp_sock_read_txrx);
        u32        tsoffset;        /* timestamp offset */
        u32        snd_wnd;        /* The window we expect to receive        */
        u32        mss_cache;        /* Cached effective mss, not including SACKS */
        u32        snd_cwnd;        /* Sending congestion window                */
        u32        prr_out;        /* Total number of pkts sent during Recovery. */
        u32        lost_out;        /* Lost packets                        */
        u32        sacked_out;        /* SACK'd packets                        */
        u16        tcp_header_len;        /* Bytes of tcp header to send                */
        u8        scaling_ratio;        /* see tcp_win_from_space() */
        u8        chrono_type : 2,        /* current chronograph type */
                repair      : 1,
                tcp_usec_ts : 1, /* TSval values in usec */
                is_sack_reneg:1,    /* in recovery from loss with SACK reneg? */
                is_cwnd_limited:1;/* forward progress limited by snd_cwnd? */
        __cacheline_group_end(tcp_sock_read_txrx);

        /* RX read-mostly hotpath cache lines */
        __cacheline_group_begin(tcp_sock_read_rx);
        u32        copied_seq;        /* Head of yet unread data */
        u32        rcv_tstamp;        /* timestamp of last received ACK (for keepalives) */
        u32        snd_wl1;        /* Sequence for window update                */
        u32        tlp_high_seq;        /* snd_nxt at the time of TLP */
        u32        rttvar_us;        /* smoothed mdev_max                        */
        u32        retrans_out;        /* Retransmitted packets out                */
        u16        advmss;                /* Advertised MSS                        */
        u16        urg_data;        /* Saved octet of OOB data and control flags */
        u32        lost;                /* Total data packets lost incl. rexmits */
        struct  minmax rtt_min;
        /* OOO segments go in this rbtree. Socket lock must be held. */
        struct rb_root        out_of_order_queue;
        u32        snd_ssthresh;        /* Slow start size threshold                */
        u8        recvmsg_inq : 1;/* Indicate # of bytes in queue upon recvmsg */
        __cacheline_group_end(tcp_sock_read_rx);

        /* TX read-write hotpath cache lines */
        __cacheline_group_begin(tcp_sock_write_tx) ____cacheline_aligned;
        u32        segs_out;        /* RFC4898 tcpEStatsPerfSegsOut
                                 * The total number of segments sent.
                                 */
        u32        data_segs_out;        /* RFC4898 tcpEStatsPerfDataSegsOut
                                 * total number of data segments sent.
                                 */
        u64        bytes_sent;        /* RFC4898 tcpEStatsPerfHCDataOctetsOut
                                 * total number of data bytes sent.
                                 */
        u32        snd_sml;        /* Last byte of the most recently transmitted small packet */
        u32        chrono_start;        /* Start time in jiffies of a TCP chrono */
        u32        chrono_stat[3];        /* Time in jiffies for chrono_stat stats */
        u32        write_seq;        /* Tail(+1) of data held in tcp send buffer */
        u32        pushed_seq;        /* Last pushed seq, required to talk to windows */
        u32        lsndtime;
        u32        mdev_us;        /* medium deviation                        */
        u32        rtt_seq;        /* sequence number to update rttvar        */
        u64        tcp_wstamp_ns;        /* departure time for next sent data packet */
        struct list_head tsorted_sent_queue; /* time-sorted sent but un-SACKed skbs */
        struct sk_buff *highest_sack;   /* skb just after the highest
                                         * skb with SACKed bit set
                                         * (validity guaranteed only if
                                         * sacked_out > 0)
                                         */
        u8        ecn_flags;        /* ECN status bits.                        */
        __cacheline_group_end(tcp_sock_write_tx);

        /* TXRX read-write hotpath cache lines */
        __cacheline_group_begin(tcp_sock_write_txrx);
/*
 *        Header prediction flags
 *        0x5?10 << 16 + snd_wnd in net byte order
 */
        __be32        pred_flags;
        u64        tcp_clock_cache; /* cache last tcp_clock_ns() (see tcp_mstamp_refresh()) */
        u64        tcp_mstamp;        /* most recent packet received/sent */
        u32        rcv_nxt;        /* What we want to receive next                */
        u32        snd_nxt;        /* Next sequence we send                */
        u32        snd_una;        /* First byte we want an ack for        */
        u32        window_clamp;        /* Maximal window to advertise                */
        u32        srtt_us;        /* smoothed round trip time << 3 in usecs */
        u32        packets_out;        /* Packets which are "in flight"        */
        u32        snd_up;                /* Urgent pointer                */
        u32        delivered;        /* Total data packets delivered incl. rexmits */
        u32        delivered_ce;        /* Like the above but only ECE marked packets */
        u32        app_limited;        /* limited until "delivered" reaches this val */
        u32        rcv_wnd;        /* Current receiver window                */
/*
 *      Options received (usually on last packet, some only on SYN packets).
 */
        struct tcp_options_received rx_opt;
        u8        nonagle     : 4,/* Disable Nagle algorithm?             */
                rate_app_limited:1;  /* rate_{delivered,interval_us} limited? */
        __cacheline_group_end(tcp_sock_write_txrx);

        /* RX read-write hotpath cache lines */
        __cacheline_group_begin(tcp_sock_write_rx) __aligned(8);
        u64        bytes_received;
                                /* RFC4898 tcpEStatsAppHCThruOctetsReceived
                                 * sum(delta(rcv_nxt)), or how many bytes
                                 * were acked.
                                 */
        u32        segs_in;        /* RFC4898 tcpEStatsPerfSegsIn
                                 * total number of segments in.
                                 */
        u32        data_segs_in;        /* RFC4898 tcpEStatsPerfDataSegsIn
                                 * total number of data segments in.
                                 */
        u32        rcv_wup;        /* rcv_nxt on last window update sent        */
        u32        max_packets_out;  /* max packets_out in last window */
        u32        cwnd_usage_seq;  /* right edge of cwnd usage tracking flight */
        u32        rate_delivered;    /* saved rate sample: packets delivered */
        u32        rate_interval_us;  /* saved rate sample: time elapsed */
        u32        rcv_rtt_last_tsecr;
        u64        first_tx_mstamp;  /* start of window send phase */
        u64        delivered_mstamp; /* time we reached "delivered" */
        u64        bytes_acked;        /* RFC4898 tcpEStatsAppHCThruOctetsAcked
                                 * sum(delta(snd_una)), or how many bytes
                                 * were acked.
                                 */
        struct {
                u32        rtt_us;
                u32        seq;
                u64        time;
        } rcv_rtt_est;
/* Receiver queue space */
        struct {
                u32        space;
                u32        seq;
                u64        time;
        } rcvq_space;
        __cacheline_group_end(tcp_sock_write_rx);
        /* End of Hot Path */

/*
 *        RFC793 variables by their proper names. This means you can
 *        read the code and the spec side by side (and laugh ...)
 *        See RFC793 and RFC1122. The RFC writes these in capitals.
 */
        u32        dsack_dups;        /* RFC4898 tcpEStatsStackDSACKDups
                                 * total number of DSACK blocks received
                                 */
        u32        compressed_ack_rcv_nxt;
        struct list_head tsq_node; /* anchor in tsq_tasklet.head list */

        /* Information of the most recently (s)acked skb */
        struct tcp_rack {
                u64 mstamp; /* (Re)sent time of the skb */
                u32 rtt_us;  /* Associated RTT */
                u32 end_seq; /* Ending TCP sequence of the skb */
                u32 last_delivered; /* tp->delivered at last reo_wnd adj */
                u8 reo_wnd_steps;   /* Allowed reordering window */
#define TCP_RACK_RECOVERY_THRESH 16
                u8 reo_wnd_persist:5, /* No. of recovery since last adj */
                   dsack_seen:1, /* Whether DSACK seen after last adj */
                   advanced:1;         /* mstamp advanced since last lost marking */
        } rack;
        u8        compressed_ack;
        u8        dup_ack_counter:2,
                tlp_retrans:1,        /* TLP is a retransmission */
                unused:5;
        u8        thin_lto    : 1,/* Use linear timeouts for thin streams */
                fastopen_connect:1, /* FASTOPEN_CONNECT sockopt */
                fastopen_no_cookie:1, /* Allow send/recv SYN+data without a cookie */
                fastopen_client_fail:2, /* reason why fastopen failed */
                frto        : 1;/* F-RTO (RFC5682) activated in CA_Loss */
        u8        repair_queue;
        u8        save_syn:2,        /* Save headers of SYN packet */
                syn_data:1,        /* SYN includes data */
                syn_fastopen:1,        /* SYN includes Fast Open option */
                syn_fastopen_exp:1,/* SYN includes Fast Open exp. option */
                syn_fastopen_ch:1, /* Active TFO re-enabling probe */
                syn_data_acked:1;/* data in SYN is acked by SYN-ACK */

        u8        keepalive_probes; /* num of allowed keep alive probes        */
        u32        tcp_tx_delay;        /* delay (in usec) added to TX packets */

/* RTT measurement */
        u32        mdev_max_us;        /* maximal mdev for the last rtt period        */

        u32        reord_seen;        /* number of data packet reordering events */

/*
 *        Slow start and congestion control (see also Nagle, and Karn & Partridge)
 */
        u32        snd_cwnd_cnt;        /* Linear increase counter                */
        u32        snd_cwnd_clamp; /* Do not allow snd_cwnd to grow above this */
        u32        snd_cwnd_used;
        u32        snd_cwnd_stamp;
        u32        prior_cwnd;        /* cwnd right before starting loss recovery */
        u32        prr_delivered;        /* Number of newly delivered packets to
                                 * receiver in Recovery. */
        u32        last_oow_ack_time;  /* timestamp of last out-of-window ACK */

        struct hrtimer        pacing_timer;
        struct hrtimer        compressed_ack_timer;

        struct sk_buff        *ooo_last_skb; /* cache rb_last(out_of_order_queue) */

        /* SACKs data, these 2 need to be together (see tcp_options_write) */
        struct tcp_sack_block duplicate_sack[1]; /* D-SACK block */
        struct tcp_sack_block selective_acks[4]; /* The SACKS themselves*/

        struct tcp_sack_block recv_sack_cache[4];

        int     lost_cnt_hint;

        u32        prior_ssthresh; /* ssthresh saved at recovery start        */
        u32        high_seq;        /* snd_nxt at onset of congestion        */

        u32        retrans_stamp;        /* Timestamp of the last retransmit,
                                 * also used in SYN-SENT to remember stamp of
                                 * the first SYN. */
        u32        undo_marker;        /* snd_una upon a new recovery episode. */
        int        undo_retrans;        /* number of undoable retransmissions. */
        u64        bytes_retrans;        /* RFC4898 tcpEStatsPerfOctetsRetrans
                                 * Total data bytes retransmitted
                                 */
        u32        total_retrans;        /* Total retransmits for entire connection */
        u32        rto_stamp;        /* Start time (ms) of last CA_Loss recovery */
        u16        total_rto;        /* Total number of RTO timeouts, including
                                 * SYN/SYN-ACK and recurring timeouts.
                                 */
        u16        total_rto_recoveries;        /* Total number of RTO recoveries,
                                         * including any unfinished recovery.
                                         */
        u32        total_rto_time;        /* ms spent in (completed) RTO recoveries. */

        u32        urg_seq;        /* Seq of received urgent pointer */
        unsigned int                keepalive_time;          /* time before keep alive takes place */
        unsigned int                keepalive_intvl;  /* time interval between keep alive probes */

        int                        linger2;


/* Sock_ops bpf program related variables */
#ifdef CONFIG_BPF
        u8        bpf_sock_ops_cb_flags;  /* Control calling BPF programs
                                         * values defined in uapi/linux/tcp.h
                                         */
        u8        bpf_chg_cc_inprogress:1; /* In the middle of
                                          * bpf_setsockopt(TCP_CONGESTION),
                                          * it is to avoid the bpf_tcp_cc->init()
                                          * to recur itself by calling
                                          * bpf_setsockopt(TCP_CONGESTION, "itself").
                                          */
#define BPF_SOCK_OPS_TEST_FLAG(TP, ARG) (TP->bpf_sock_ops_cb_flags & ARG)
#else
#define BPF_SOCK_OPS_TEST_FLAG(TP, ARG) 0
#endif

        u16 timeout_rehash;        /* Timeout-triggered rehash attempts */

        u32 rcv_ooopack; /* Received out-of-order packets, for tcpinfo */

/* TCP-specific MTU probe information. */
        struct {
                u32                  probe_seq_start;
                u32                  probe_seq_end;
        } mtu_probe;
        u32     plb_rehash;     /* PLB-triggered rehash attempts */
        u32        mtu_info; /* We received an ICMP_FRAG_NEEDED / ICMPV6_PKT_TOOBIG
                           * while socket was owned by user.
                           */
#if IS_ENABLED(CONFIG_MPTCP)
        bool        is_mptcp;
#endif
#if IS_ENABLED(CONFIG_SMC)
        bool        syn_smc;        /* SYN includes SMC */
        bool        (*smc_hs_congested)(const struct sock *sk);
#endif

#if defined(CONFIG_TCP_MD5SIG) || defined(CONFIG_TCP_AO)
/* TCP AF-Specific parts; only used by TCP-AO/MD5 Signature support so far */
        const struct tcp_sock_af_ops        *af_specific;

#ifdef CONFIG_TCP_MD5SIG
/* TCP MD5 Signature Option information */
        struct tcp_md5sig_info        __rcu *md5sig_info;
#endif
#ifdef CONFIG_TCP_AO
        struct tcp_ao_info        __rcu *ao_info;
#endif
#endif

/* TCP fastopen related information */
        struct tcp_fastopen_request *fastopen_req;
        /* fastopen_rsk points to request_sock that resulted in this big
         * socket. Used to retransmit SYNACKs etc.
         */
        struct request_sock __rcu *fastopen_rsk;
        struct saved_syn *saved_syn;
};

enum tsq_enum {
        TSQ_THROTTLED,
        TSQ_QUEUED,
        TCP_TSQ_DEFERRED,           /* tcp_tasklet_func() found socket was owned */
        TCP_WRITE_TIMER_DEFERRED,  /* tcp_write_timer() found socket was owned */
        TCP_DELACK_TIMER_DEFERRED, /* tcp_delack_timer() found socket was owned */
        TCP_MTU_REDUCED_DEFERRED,  /* tcp_v{4|6}_err() could not call
                                    * tcp_v{4|6}_mtu_reduced()
                                    */
        TCP_ACK_DEFERRED,           /* TX pure ack is deferred */
};

enum tsq_flags {
        TSQF_THROTTLED                        = BIT(TSQ_THROTTLED),
        TSQF_QUEUED                        = BIT(TSQ_QUEUED),
        TCPF_TSQ_DEFERRED                = BIT(TCP_TSQ_DEFERRED),
        TCPF_WRITE_TIMER_DEFERRED        = BIT(TCP_WRITE_TIMER_DEFERRED),
        TCPF_DELACK_TIMER_DEFERRED        = BIT(TCP_DELACK_TIMER_DEFERRED),
        TCPF_MTU_REDUCED_DEFERRED        = BIT(TCP_MTU_REDUCED_DEFERRED),
        TCPF_ACK_DEFERRED                = BIT(TCP_ACK_DEFERRED),
};

#define tcp_sk(ptr) container_of_const(ptr, struct tcp_sock, inet_conn.icsk_inet.sk)

/* Variant of tcp_sk() upgrading a const sock to a read/write tcp socket.
 * Used in context of (lockless) tcp listeners.
 */
#define tcp_sk_rw(ptr) container_of(ptr, struct tcp_sock, inet_conn.icsk_inet.sk)

struct tcp_timewait_sock {
        struct inet_timewait_sock tw_sk;
#define tw_rcv_nxt tw_sk.__tw_common.skc_tw_rcv_nxt
#define tw_snd_nxt tw_sk.__tw_common.skc_tw_snd_nxt
        u32                          tw_rcv_wnd;
        u32                          tw_ts_offset;
        u32                          tw_ts_recent;

        /* The time we sent the last out-of-window ACK: */
        u32                          tw_last_oow_ack_time;

        int                          tw_ts_recent_stamp;
        u32                          tw_tx_delay;
#ifdef CONFIG_TCP_MD5SIG
        struct tcp_md5sig_key          *tw_md5_key;
#endif
#ifdef CONFIG_TCP_AO
        struct tcp_ao_info        __rcu *ao_info;
#endif
};

static inline struct tcp_timewait_sock *tcp_twsk(const struct sock *sk)
{
        return (struct tcp_timewait_sock *)sk;
}

static inline bool tcp_passive_fastopen(const struct sock *sk)
{
        return sk->sk_state == TCP_SYN_RECV &&
               rcu_access_pointer(tcp_sk(sk)->fastopen_rsk) != NULL;
}

static inline void fastopen_queue_tune(struct sock *sk, int backlog)
{
        struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
        int somaxconn = READ_ONCE(sock_net(sk)->core.sysctl_somaxconn);

        WRITE_ONCE(queue->fastopenq.max_qlen, min_t(unsigned int, backlog, somaxconn));
}

static inline void tcp_move_syn(struct tcp_sock *tp,
                                struct request_sock *req)
{
        tp->saved_syn = req->saved_syn;
        req->saved_syn = NULL;
}

static inline void tcp_saved_syn_free(struct tcp_sock *tp)
{
        kfree(tp->saved_syn);
        tp->saved_syn = NULL;
}

static inline u32 tcp_saved_syn_len(const struct saved_syn *saved_syn)
{
        return saved_syn->mac_hdrlen + saved_syn->network_hdrlen +
                saved_syn->tcp_hdrlen;
}

struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk,
                                               const struct sk_buff *orig_skb,
                                               const struct sk_buff *ack_skb);

static inline u16 tcp_mss_clamp(const struct tcp_sock *tp, u16 mss)
{
        /* We use READ_ONCE() here because socket might not be locked.
         * This happens for listeners.
         */
        u16 user_mss = READ_ONCE(tp->rx_opt.user_mss);

        return (user_mss && user_mss < mss) ? user_mss : mss;
}

int tcp_skb_shift(struct sk_buff *to, struct sk_buff *from, int pcount,
                  int shiftlen);

void __tcp_sock_set_cork(struct sock *sk, bool on);
void tcp_sock_set_cork(struct sock *sk, bool on);
int tcp_sock_set_keepcnt(struct sock *sk, int val);
int tcp_sock_set_keepidle_locked(struct sock *sk, int val);
int tcp_sock_set_keepidle(struct sock *sk, int val);
int tcp_sock_set_keepintvl(struct sock *sk, int val);
void __tcp_sock_set_nodelay(struct sock *sk, bool on);
void tcp_sock_set_nodelay(struct sock *sk);
void tcp_sock_set_quickack(struct sock *sk, int val);
int tcp_sock_set_syncnt(struct sock *sk, int val);
int tcp_sock_set_user_timeout(struct sock *sk, int val);

static inline bool dst_tcp_usec_ts(const struct dst_entry *dst)
{
        return dst_feature(dst, RTAX_FEATURE_TCP_USEC_TS);
}

#endif        /* _LINUX_TCP_H */















































































































   70 







   58 

   37 

















   57 






















   57 










   45 













   57 




   57 














   57 










    5 





    5 



    5 

    5 























    3 


    3 

    3 

    3 


    3 


















   16 


   16 

   14 











    1 













    3 
























    2 



    1 
    2 




   13 




   14 




   14 


    6 
    1 






   28 
   28 


    1 













    1 



    2 


    2 























    3 





    3 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
/*
 * Copyright (c) 2007, 2017 Oracle and/or its affiliates. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/rbtree.h>
#include <linux/bitops.h>
#include <linux/export.h>

#include "rds.h"

/*
 * This file implements the receive side of the unconventional congestion
 * management in RDS.
 *
 * Messages waiting in the receive queue on the receiving socket are accounted
 * against the sockets SO_RCVBUF option value.  Only the payload bytes in the
 * message are accounted for.  If the number of bytes queued equals or exceeds
 * rcvbuf then the socket is congested.  All sends attempted to this socket's
 * address should return block or return -EWOULDBLOCK.
 *
 * Applications are expected to be reasonably tuned such that this situation
 * very rarely occurs.  An application encountering this "back-pressure" is
 * considered a bug.
 *
 * This is implemented by having each node maintain bitmaps which indicate
 * which ports on bound addresses are congested.  As the bitmap changes it is
 * sent through all the connections which terminate in the local address of the
 * bitmap which changed.
 *
 * The bitmaps are allocated as connections are brought up.  This avoids
 * allocation in the interrupt handling path which queues messages on sockets.
 * The dense bitmaps let transports send the entire bitmap on any bitmap change
 * reasonably efficiently.  This is much easier to implement than some
 * finer-grained communication of per-port congestion.  The sender does a very
 * inexpensive bit test to test if the port it's about to send to is congested
 * or not.
 */

/*
 * Interaction with poll is a tad tricky. We want all processes stuck in
 * poll to wake up and check whether a congested destination became uncongested.
 * The really sad thing is we have no idea which destinations the application
 * wants to send to - we don't even know which rds_connections are involved.
 * So until we implement a more flexible rds poll interface, we have to make
 * do with this:
 * We maintain a global counter that is incremented each time a congestion map
 * update is received. Each rds socket tracks this value, and if rds_poll
 * finds that the saved generation number is smaller than the global generation
 * number, it wakes up the process.
 */
static atomic_t                rds_cong_generation = ATOMIC_INIT(0);

/*
 * Congestion monitoring
 */
static LIST_HEAD(rds_cong_monitor);
static DEFINE_RWLOCK(rds_cong_monitor_lock);

/*
 * Yes, a global lock.  It's used so infrequently that it's worth keeping it
 * global to simplify the locking.  It's only used in the following
 * circumstances:
 *
 *  - on connection buildup to associate a conn with its maps
 *  - on map changes to inform conns of a new map to send
 *
 *  It's sadly ordered under the socket callback lock and the connection lock.
 *  Receive paths can mark ports congested from interrupt context so the
 *  lock masks interrupts.
 */
static DEFINE_SPINLOCK(rds_cong_lock);
static struct rb_root rds_cong_tree = RB_ROOT;

static struct rds_cong_map *rds_cong_tree_walk(const struct in6_addr *addr,
                                               struct rds_cong_map *insert)
{
        struct rb_node **p = &rds_cong_tree.rb_node;
        struct rb_node *parent = NULL;
        struct rds_cong_map *map;

        while (*p) {
                int diff;

                parent = *p;
                map = rb_entry(parent, struct rds_cong_map, m_rb_node);

                diff = rds_addr_cmp(addr, &map->m_addr);
                if (diff < 0)
                        p = &(*p)->rb_left;
                else if (diff > 0)
                        p = &(*p)->rb_right;
                else
                        return map;
        }

        if (insert) {
                rb_link_node(&insert->m_rb_node, parent, p);
                rb_insert_color(&insert->m_rb_node, &rds_cong_tree);
        }
        return NULL;
}

/*
 * There is only ever one bitmap for any address.  Connections try and allocate
 * these bitmaps in the process getting pointers to them.  The bitmaps are only
 * ever freed as the module is removed after all connections have been freed.
 */
static struct rds_cong_map *rds_cong_from_addr(const struct in6_addr *addr)
{
        struct rds_cong_map *map;
        struct rds_cong_map *ret = NULL;
        unsigned long zp;
        unsigned long i;
        unsigned long flags;

        map = kzalloc(sizeof(struct rds_cong_map), GFP_KERNEL);
        if (!map)
                return NULL;

        map->m_addr = *addr;
        init_waitqueue_head(&map->m_waitq);
        INIT_LIST_HEAD(&map->m_conn_list);

        for (i = 0; i < RDS_CONG_MAP_PAGES; i++) {
                zp = get_zeroed_page(GFP_KERNEL);
                if (zp == 0)
                        goto out;
                map->m_page_addrs[i] = zp;
        }

        spin_lock_irqsave(&rds_cong_lock, flags);
        ret = rds_cong_tree_walk(addr, map);
        spin_unlock_irqrestore(&rds_cong_lock, flags);

        if (!ret) {
                ret = map;
                map = NULL;
        }

out:
        if (map) {
                for (i = 0; i < RDS_CONG_MAP_PAGES && map->m_page_addrs[i]; i++)
                        free_page(map->m_page_addrs[i]);
                kfree(map);
        }

        rdsdebug("map %p for addr %pI6c\n", ret, addr);

        return ret;
}

/*
 * Put the conn on its local map's list.  This is called when the conn is
 * really added to the hash.  It's nested under the rds_conn_lock, sadly.
 */
void rds_cong_add_conn(struct rds_connection *conn)
{
        unsigned long flags;

        rdsdebug("conn %p now on map %p\n", conn, conn->c_lcong);
        spin_lock_irqsave(&rds_cong_lock, flags);
        list_add_tail(&conn->c_map_item, &conn->c_lcong->m_conn_list);
        spin_unlock_irqrestore(&rds_cong_lock, flags);
}

void rds_cong_remove_conn(struct rds_connection *conn)
{
        unsigned long flags;

        rdsdebug("removing conn %p from map %p\n", conn, conn->c_lcong);
        spin_lock_irqsave(&rds_cong_lock, flags);
        list_del_init(&conn->c_map_item);
        spin_unlock_irqrestore(&rds_cong_lock, flags);
}

int rds_cong_get_maps(struct rds_connection *conn)
{
        conn->c_lcong = rds_cong_from_addr(&conn->c_laddr);
        conn->c_fcong = rds_cong_from_addr(&conn->c_faddr);

        if (!(conn->c_lcong && conn->c_fcong))
                return -ENOMEM;

        return 0;
}

void rds_cong_queue_updates(struct rds_cong_map *map)
{
        struct rds_connection *conn;
        unsigned long flags;

        spin_lock_irqsave(&rds_cong_lock, flags);

        list_for_each_entry(conn, &map->m_conn_list, c_map_item) {
                struct rds_conn_path *cp = &conn->c_path[0];

                rcu_read_lock();
                if (!test_and_set_bit(0, &conn->c_map_queued) &&
                    !rds_destroy_pending(cp->cp_conn)) {
                        rds_stats_inc(s_cong_update_queued);
                        /* We cannot inline the call to rds_send_xmit() here
                         * for two reasons (both pertaining to a TCP transport):
                         * 1. When we get here from the receive path, we
                         *    are already holding the sock_lock (held by
                         *    tcp_v4_rcv()). So inlining calls to
                         *    tcp_setsockopt and/or tcp_sendmsg will deadlock
                         *    when it tries to get the sock_lock())
                         * 2. Interrupts are masked so that we can mark the
                         *    port congested from both send and recv paths.
                         *    (See comment around declaration of rdc_cong_lock).
                         *    An attempt to get the sock_lock() here will
                         *    therefore trigger warnings.
                         * Defer the xmit to rds_send_worker() instead.
                         */
                        queue_delayed_work(rds_wq, &cp->cp_send_w, 0);
                }
                rcu_read_unlock();
        }

        spin_unlock_irqrestore(&rds_cong_lock, flags);
}

void rds_cong_map_updated(struct rds_cong_map *map, uint64_t portmask)
{
        rdsdebug("waking map %p for %pI4\n",
          map, &map->m_addr);
        rds_stats_inc(s_cong_update_received);
        atomic_inc(&rds_cong_generation);
        if (waitqueue_active(&map->m_waitq))
                wake_up(&map->m_waitq);
        if (waitqueue_active(&rds_poll_waitq))
                wake_up_all(&rds_poll_waitq);

        if (portmask && !list_empty(&rds_cong_monitor)) {
                unsigned long flags;
                struct rds_sock *rs;

                read_lock_irqsave(&rds_cong_monitor_lock, flags);
                list_for_each_entry(rs, &rds_cong_monitor, rs_cong_list) {
                        spin_lock(&rs->rs_lock);
                        rs->rs_cong_notify |= (rs->rs_cong_mask & portmask);
                        rs->rs_cong_mask &= ~portmask;
                        spin_unlock(&rs->rs_lock);
                        if (rs->rs_cong_notify)
                                rds_wake_sk_sleep(rs);
                }
                read_unlock_irqrestore(&rds_cong_monitor_lock, flags);
        }
}
EXPORT_SYMBOL_GPL(rds_cong_map_updated);

int rds_cong_updated_since(unsigned long *recent)
{
        unsigned long gen = atomic_read(&rds_cong_generation);

        if (likely(*recent == gen))
                return 0;
        *recent = gen;
        return 1;
}

/*
 * We're called under the locking that protects the sockets receive buffer
 * consumption.  This makes it a lot easier for the caller to only call us
 * when it knows that an existing set bit needs to be cleared, and vice versa.
 * We can't block and we need to deal with concurrent sockets working against
 * the same per-address map.
 */
void rds_cong_set_bit(struct rds_cong_map *map, __be16 port)
{
        unsigned long i;
        unsigned long off;

        rdsdebug("setting congestion for %pI4:%u in map %p\n",
          &map->m_addr, ntohs(port), map);

        i = be16_to_cpu(port) / RDS_CONG_MAP_PAGE_BITS;
        off = be16_to_cpu(port) % RDS_CONG_MAP_PAGE_BITS;

        set_bit_le(off, (void *)map->m_page_addrs[i]);
}

void rds_cong_clear_bit(struct rds_cong_map *map, __be16 port)
{
        unsigned long i;
        unsigned long off;

        rdsdebug("clearing congestion for %pI4:%u in map %p\n",
          &map->m_addr, ntohs(port), map);

        i = be16_to_cpu(port) / RDS_CONG_MAP_PAGE_BITS;
        off = be16_to_cpu(port) % RDS_CONG_MAP_PAGE_BITS;

        clear_bit_le(off, (void *)map->m_page_addrs[i]);
}

static int rds_cong_test_bit(struct rds_cong_map *map, __be16 port)
{
        unsigned long i;
        unsigned long off;

        i = be16_to_cpu(port) / RDS_CONG_MAP_PAGE_BITS;
        off = be16_to_cpu(port) % RDS_CONG_MAP_PAGE_BITS;

        return test_bit_le(off, (void *)map->m_page_addrs[i]);
}

void rds_cong_add_socket(struct rds_sock *rs)
{
        unsigned long flags;

        write_lock_irqsave(&rds_cong_monitor_lock, flags);
        if (list_empty(&rs->rs_cong_list))
                list_add(&rs->rs_cong_list, &rds_cong_monitor);
        write_unlock_irqrestore(&rds_cong_monitor_lock, flags);
}

void rds_cong_remove_socket(struct rds_sock *rs)
{
        unsigned long flags;
        struct rds_cong_map *map;

        write_lock_irqsave(&rds_cong_monitor_lock, flags);
        list_del_init(&rs->rs_cong_list);
        write_unlock_irqrestore(&rds_cong_monitor_lock, flags);

        /* update congestion map for now-closed port */
        spin_lock_irqsave(&rds_cong_lock, flags);
        map = rds_cong_tree_walk(&rs->rs_bound_addr, NULL);
        spin_unlock_irqrestore(&rds_cong_lock, flags);

        if (map && rds_cong_test_bit(map, rs->rs_bound_port)) {
                rds_cong_clear_bit(map, rs->rs_bound_port);
                rds_cong_queue_updates(map);
        }
}

int rds_cong_wait(struct rds_cong_map *map, __be16 port, int nonblock,
                  struct rds_sock *rs)
{
        if (!rds_cong_test_bit(map, port))
                return 0;
        if (nonblock) {
                if (rs && rs->rs_cong_monitor) {
                        unsigned long flags;

                        /* It would have been nice to have an atomic set_bit on
                         * a uint64_t. */
                        spin_lock_irqsave(&rs->rs_lock, flags);
                        rs->rs_cong_mask |= RDS_CONG_MONITOR_MASK(ntohs(port));
                        spin_unlock_irqrestore(&rs->rs_lock, flags);

                        /* Test again - a congestion update may have arrived in
                         * the meantime. */
                        if (!rds_cong_test_bit(map, port))
                                return 0;
                }
                rds_stats_inc(s_cong_send_error);
                return -ENOBUFS;
        }

        rds_stats_inc(s_cong_send_blocked);
        rdsdebug("waiting on map %p for port %u\n", map, be16_to_cpu(port));

        return wait_event_interruptible(map->m_waitq,
                                        !rds_cong_test_bit(map, port));
}

void rds_cong_exit(void)
{
        struct rb_node *node;
        struct rds_cong_map *map;
        unsigned long i;

        while ((node = rb_first(&rds_cong_tree))) {
                map = rb_entry(node, struct rds_cong_map, m_rb_node);
                rdsdebug("freeing map %p\n", map);
                rb_erase(&map->m_rb_node, &rds_cong_tree);
                for (i = 0; i < RDS_CONG_MAP_PAGES && map->m_page_addrs[i]; i++)
                        free_page(map->m_page_addrs[i]);
                kfree(map);
        }
}

/*
 * Allocate a RDS message containing a congestion update.
 */
struct rds_message *rds_cong_update_alloc(struct rds_connection *conn)
{
        struct rds_cong_map *map = conn->c_lcong;
        struct rds_message *rm;

        rm = rds_message_map_pages(map->m_page_addrs, RDS_CONG_MAP_BYTES);
        if (!IS_ERR(rm))
                rm->m_inc.i_hdr.h_flags = RDS_FLAG_CONG_BITMAP;

        return rm;
}











































































































  463 








  254 





























































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Statically sized hash table implementation
 * (C) 2012  Sasha Levin <levinsasha928@gmail.com>
 */

#ifndef _LINUX_HASHTABLE_H
#define _LINUX_HASHTABLE_H

#include <linux/list.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/hash.h>
#include <linux/rculist.h>

#define DEFINE_HASHTABLE(name, bits)                                                \
        struct hlist_head name[1 << (bits)] =                                        \
                        { [0 ... ((1 << (bits)) - 1)] = HLIST_HEAD_INIT }

#define DEFINE_READ_MOSTLY_HASHTABLE(name, bits)                                \
        struct hlist_head name[1 << (bits)] __read_mostly =                        \
                        { [0 ... ((1 << (bits)) - 1)] = HLIST_HEAD_INIT }

#define DECLARE_HASHTABLE(name, bits)                                           \
        struct hlist_head name[1 << (bits)]

#define HASH_SIZE(name) (ARRAY_SIZE(name))
#define HASH_BITS(name) ilog2(HASH_SIZE(name))

/* Use hash_32 when possible to allow for fast 32bit hashing in 64bit kernels. */
#define hash_min(val, bits)                                                        \
        (sizeof(val) <= 4 ? hash_32(val, bits) : hash_long(val, bits))

static inline void __hash_init(struct hlist_head *ht, unsigned int sz)
{
        unsigned int i;

        for (i = 0; i < sz; i++)
                INIT_HLIST_HEAD(&ht[i]);
}

/**
 * hash_init - initialize a hash table
 * @hashtable: hashtable to be initialized
 *
 * Calculates the size of the hashtable from the given parameter, otherwise
 * same as hash_init_size.
 *
 * This has to be a macro since HASH_BITS() will not work on pointers since
 * it calculates the size during preprocessing.
 */
#define hash_init(hashtable) __hash_init(hashtable, HASH_SIZE(hashtable))

/**
 * hash_add - add an object to a hashtable
 * @hashtable: hashtable to add to
 * @node: the &struct hlist_node of the object to be added
 * @key: the key of the object to be added
 */
#define hash_add(hashtable, node, key)                                                \
        hlist_add_head(node, &hashtable[hash_min(key, HASH_BITS(hashtable))])

/**
 * hash_add_rcu - add an object to a rcu enabled hashtable
 * @hashtable: hashtable to add to
 * @node: the &struct hlist_node of the object to be added
 * @key: the key of the object to be added
 */
#define hash_add_rcu(hashtable, node, key)                                        \
        hlist_add_head_rcu(node, &hashtable[hash_min(key, HASH_BITS(hashtable))])

/**
 * hash_hashed - check whether an object is in any hashtable
 * @node: the &struct hlist_node of the object to be checked
 */
static inline bool hash_hashed(struct hlist_node *node)
{
        return !hlist_unhashed(node);
}

static inline bool __hash_empty(struct hlist_head *ht, unsigned int sz)
{
        unsigned int i;

        for (i = 0; i < sz; i++)
                if (!hlist_empty(&ht[i]))
                        return false;

        return true;
}

/**
 * hash_empty - check whether a hashtable is empty
 * @hashtable: hashtable to check
 *
 * This has to be a macro since HASH_BITS() will not work on pointers since
 * it calculates the size during preprocessing.
 */
#define hash_empty(hashtable) __hash_empty(hashtable, HASH_SIZE(hashtable))

/**
 * hash_del - remove an object from a hashtable
 * @node: &struct hlist_node of the object to remove
 */
static inline void hash_del(struct hlist_node *node)
{
        hlist_del_init(node);
}

/**
 * hash_del_rcu - remove an object from a rcu enabled hashtable
 * @node: &struct hlist_node of the object to remove
 */
static inline void hash_del_rcu(struct hlist_node *node)
{
        hlist_del_init_rcu(node);
}

/**
 * hash_for_each - iterate over a hashtable
 * @name: hashtable to iterate
 * @bkt: integer to use as bucket loop cursor
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 */
#define hash_for_each(name, bkt, obj, member)                                \
        for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
                        (bkt)++)\
                hlist_for_each_entry(obj, &name[bkt], member)

/**
 * hash_for_each_rcu - iterate over a rcu enabled hashtable
 * @name: hashtable to iterate
 * @bkt: integer to use as bucket loop cursor
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 */
#define hash_for_each_rcu(name, bkt, obj, member)                        \
        for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
                        (bkt)++)\
                hlist_for_each_entry_rcu(obj, &name[bkt], member)

/**
 * hash_for_each_safe - iterate over a hashtable safe against removal of
 * hash entry
 * @name: hashtable to iterate
 * @bkt: integer to use as bucket loop cursor
 * @tmp: a &struct hlist_node used for temporary storage
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 */
#define hash_for_each_safe(name, bkt, tmp, obj, member)                        \
        for ((bkt) = 0, obj = NULL; obj == NULL && (bkt) < HASH_SIZE(name);\
                        (bkt)++)\
                hlist_for_each_entry_safe(obj, tmp, &name[bkt], member)

/**
 * hash_for_each_possible - iterate over all possible objects hashing to the
 * same bucket
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 */
#define hash_for_each_possible(name, obj, member, key)                        \
        hlist_for_each_entry(obj, &name[hash_min(key, HASH_BITS(name))], member)

/**
 * hash_for_each_possible_rcu - iterate over all possible objects hashing to the
 * same bucket in an rcu enabled hashtable
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 */
#define hash_for_each_possible_rcu(name, obj, member, key, cond...)        \
        hlist_for_each_entry_rcu(obj, &name[hash_min(key, HASH_BITS(name))],\
                member, ## cond)

/**
 * hash_for_each_possible_rcu_notrace - iterate over all possible objects hashing
 * to the same bucket in an rcu enabled hashtable in a rcu enabled hashtable
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 *
 * This is the same as hash_for_each_possible_rcu() except that it does
 * not do any RCU debugging or tracing.
 */
#define hash_for_each_possible_rcu_notrace(name, obj, member, key) \
        hlist_for_each_entry_rcu_notrace(obj, \
                &name[hash_min(key, HASH_BITS(name))], member)

/**
 * hash_for_each_possible_safe - iterate over all possible objects hashing to the
 * same bucket safe against removals
 * @name: hashtable to iterate
 * @obj: the type * to use as a loop cursor for each entry
 * @tmp: a &struct hlist_node used for temporary storage
 * @member: the name of the hlist_node within the struct
 * @key: the key of the objects to iterate over
 */
#define hash_for_each_possible_safe(name, obj, tmp, member, key)        \
        hlist_for_each_entry_safe(obj, tmp,\
                &name[hash_min(key, HASH_BITS(name))], member)


#endif











































































   30 


















   54 
   25 


   38 














    8 
    8 

    8 




















































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































































   31 


   39 



























  137 


























   15 














   12 

    3 






   13 

    2 







   12 
    3 






   15 





























  145 


















    8 


















    8 






    8 








   22 
   12 
   12 


   11 

    8 



















































  175 






  167 
   22 


  168 



  162 



  164 



  163 



















  165 
























































    2 





    2 


    4 
    2 
    2 



    3 
    2 
    1 





















    8 


   18 



    8 























    8 
    8 


    8 




















    8 
    8 























































   23 


   23 


















   20 


   20 
    8 






























    8 






    8 


    8 


    8 
    8 


    8 










    8 
















    8 


    8 

    8 





    8 

















    8 








    8 
    8 
















































































































































































































































































































































































































































































































































































































































































    7 











    8 



    8 





















    8 

















   30 
    3 





















   27 







    2 



















   25 













































































    8 


    8 
    8 

    7 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
/*
 * Copyright © 1997-2003 by The XFree86 Project, Inc.
 * Copyright © 2007 Dave Airlie
 * Copyright © 2007-2008 Intel Corporation
 *   Jesse Barnes <jesse.barnes@intel.com>
 * Copyright 2005-2006 Luc Verhaegen
 * Copyright (c) 2001, Andy Ritger  aritger@nvidia.com
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Except as contained in this notice, the name of the copyright holder(s)
 * and author(s) shall not be used in advertising or otherwise to promote
 * the sale, use or other dealings in this Software without prior written
 * authorization from the copyright holder(s) and author(s).
 */

#include <linux/ctype.h>
#include <linux/export.h>
#include <linux/fb.h> /* for KHZ2PICOS() */
#include <linux/list.h>
#include <linux/list_sort.h>
#include <linux/of.h>

#include <video/of_display_timing.h>
#include <video/of_videomode.h>
#include <video/videomode.h>

#include <drm/drm_crtc.h>
#include <drm/drm_device.h>
#include <drm/drm_edid.h>
#include <drm/drm_modes.h>
#include <drm/drm_print.h>

#include "drm_crtc_internal.h"

/**
 * drm_mode_debug_printmodeline - print a mode to dmesg
 * @mode: mode to print
 *
 * Describe @mode using DRM_DEBUG.
 */
void drm_mode_debug_printmodeline(const struct drm_display_mode *mode)
{
        DRM_DEBUG_KMS("Modeline " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
}
EXPORT_SYMBOL(drm_mode_debug_printmodeline);

/**
 * drm_mode_create - create a new display mode
 * @dev: DRM device
 *
 * Create a new, cleared drm_display_mode with kzalloc, allocate an ID for it
 * and return it.
 *
 * Returns:
 * Pointer to new mode on success, NULL on error.
 */
struct drm_display_mode *drm_mode_create(struct drm_device *dev)
{
        struct drm_display_mode *nmode;

        nmode = kzalloc(sizeof(struct drm_display_mode), GFP_KERNEL);
        if (!nmode)
                return NULL;

        return nmode;
}
EXPORT_SYMBOL(drm_mode_create);

/**
 * drm_mode_destroy - remove a mode
 * @dev: DRM device
 * @mode: mode to remove
 *
 * Release @mode's unique ID, then free it @mode structure itself using kfree.
 */
void drm_mode_destroy(struct drm_device *dev, struct drm_display_mode *mode)
{
        if (!mode)
                return;

        kfree(mode);
}
EXPORT_SYMBOL(drm_mode_destroy);

/**
 * drm_mode_probed_add - add a mode to a connector's probed_mode list
 * @connector: connector the new mode
 * @mode: mode data
 *
 * Add @mode to @connector's probed_mode list for later use. This list should
 * then in a second step get filtered and all the modes actually supported by
 * the hardware moved to the @connector's modes list.
 */
void drm_mode_probed_add(struct drm_connector *connector,
                         struct drm_display_mode *mode)
{
        WARN_ON(!mutex_is_locked(&connector->dev->mode_config.mutex));

        list_add_tail(&mode->head, &connector->probed_modes);
}
EXPORT_SYMBOL(drm_mode_probed_add);

enum drm_mode_analog {
        DRM_MODE_ANALOG_NTSC, /* 525 lines, 60Hz */
        DRM_MODE_ANALOG_PAL, /* 625 lines, 50Hz */
};

/*
 * The timings come from:
 * - https://web.archive.org/web/20220406232708/http://www.kolumbus.fi/pami1/video/pal_ntsc.html
 * - https://web.archive.org/web/20220406124914/http://martin.hinner.info/vga/pal.html
 * - https://web.archive.org/web/20220609202433/http://www.batsocks.co.uk/readme/video_timing.htm
 */
#define NTSC_LINE_DURATION_NS                63556U
#define NTSC_LINES_NUMBER                525

#define NTSC_HBLK_DURATION_TYP_NS        10900U
#define NTSC_HBLK_DURATION_MIN_NS        (NTSC_HBLK_DURATION_TYP_NS - 200)
#define NTSC_HBLK_DURATION_MAX_NS        (NTSC_HBLK_DURATION_TYP_NS + 200)

#define NTSC_HACT_DURATION_TYP_NS        (NTSC_LINE_DURATION_NS - NTSC_HBLK_DURATION_TYP_NS)
#define NTSC_HACT_DURATION_MIN_NS        (NTSC_LINE_DURATION_NS - NTSC_HBLK_DURATION_MAX_NS)
#define NTSC_HACT_DURATION_MAX_NS        (NTSC_LINE_DURATION_NS - NTSC_HBLK_DURATION_MIN_NS)

#define NTSC_HFP_DURATION_TYP_NS        1500
#define NTSC_HFP_DURATION_MIN_NS        1270
#define NTSC_HFP_DURATION_MAX_NS        2220

#define NTSC_HSLEN_DURATION_TYP_NS        4700
#define NTSC_HSLEN_DURATION_MIN_NS        (NTSC_HSLEN_DURATION_TYP_NS - 100)
#define NTSC_HSLEN_DURATION_MAX_NS        (NTSC_HSLEN_DURATION_TYP_NS + 100)

#define NTSC_HBP_DURATION_TYP_NS        4700

/*
 * I couldn't find the actual tolerance for the back porch, so let's
 * just reuse the sync length ones.
 */
#define NTSC_HBP_DURATION_MIN_NS        (NTSC_HBP_DURATION_TYP_NS - 100)
#define NTSC_HBP_DURATION_MAX_NS        (NTSC_HBP_DURATION_TYP_NS + 100)

#define PAL_LINE_DURATION_NS                64000U
#define PAL_LINES_NUMBER                625

#define PAL_HACT_DURATION_TYP_NS        51950U
#define PAL_HACT_DURATION_MIN_NS        (PAL_HACT_DURATION_TYP_NS - 100)
#define PAL_HACT_DURATION_MAX_NS        (PAL_HACT_DURATION_TYP_NS + 400)

#define PAL_HBLK_DURATION_TYP_NS        (PAL_LINE_DURATION_NS - PAL_HACT_DURATION_TYP_NS)
#define PAL_HBLK_DURATION_MIN_NS        (PAL_LINE_DURATION_NS - PAL_HACT_DURATION_MAX_NS)
#define PAL_HBLK_DURATION_MAX_NS        (PAL_LINE_DURATION_NS - PAL_HACT_DURATION_MIN_NS)

#define PAL_HFP_DURATION_TYP_NS                1650
#define PAL_HFP_DURATION_MIN_NS                (PAL_HFP_DURATION_TYP_NS - 100)
#define PAL_HFP_DURATION_MAX_NS                (PAL_HFP_DURATION_TYP_NS + 400)

#define PAL_HSLEN_DURATION_TYP_NS        4700
#define PAL_HSLEN_DURATION_MIN_NS        (PAL_HSLEN_DURATION_TYP_NS - 200)
#define PAL_HSLEN_DURATION_MAX_NS        (PAL_HSLEN_DURATION_TYP_NS + 200)

#define PAL_HBP_DURATION_TYP_NS                5700
#define PAL_HBP_DURATION_MIN_NS                (PAL_HBP_DURATION_TYP_NS - 200)
#define PAL_HBP_DURATION_MAX_NS                (PAL_HBP_DURATION_TYP_NS + 200)

struct analog_param_field {
        unsigned int even, odd;
};

#define PARAM_FIELD(_odd, _even)                \
        { .even = _even, .odd = _odd }

struct analog_param_range {
        unsigned int        min, typ, max;
};

#define PARAM_RANGE(_min, _typ, _max)                \
        { .min = _min, .typ = _typ, .max = _max }

struct analog_parameters {
        unsigned int                        num_lines;
        unsigned int                        line_duration_ns;

        struct analog_param_range        hact_ns;
        struct analog_param_range        hfp_ns;
        struct analog_param_range        hslen_ns;
        struct analog_param_range        hbp_ns;
        struct analog_param_range        hblk_ns;

        unsigned int                        bt601_hfp;

        struct analog_param_field        vfp_lines;
        struct analog_param_field        vslen_lines;
        struct analog_param_field        vbp_lines;
};

#define TV_MODE_PARAMETER(_mode, _lines, _line_dur, _hact, _hfp,        \
                          _hslen, _hbp, _hblk, _bt601_hfp, _vfp,        \
                          _vslen, _vbp)                                        \
        [_mode] = {                                                        \
                .num_lines = _lines,                                        \
                .line_duration_ns = _line_dur,                                \
                .hact_ns = _hact,                                        \
                .hfp_ns = _hfp,                                                \
                .hslen_ns = _hslen,                                        \
                .hbp_ns = _hbp,                                                \
                .hblk_ns = _hblk,                                        \
                .bt601_hfp = _bt601_hfp,                                \
                .vfp_lines = _vfp,                                        \
                .vslen_lines = _vslen,                                        \
                .vbp_lines = _vbp,                                        \
        }

static const struct analog_parameters tv_modes_parameters[] = {
        TV_MODE_PARAMETER(DRM_MODE_ANALOG_NTSC,
                          NTSC_LINES_NUMBER,
                          NTSC_LINE_DURATION_NS,
                          PARAM_RANGE(NTSC_HACT_DURATION_MIN_NS,
                                      NTSC_HACT_DURATION_TYP_NS,
                                      NTSC_HACT_DURATION_MAX_NS),
                          PARAM_RANGE(NTSC_HFP_DURATION_MIN_NS,
                                      NTSC_HFP_DURATION_TYP_NS,
                                      NTSC_HFP_DURATION_MAX_NS),
                          PARAM_RANGE(NTSC_HSLEN_DURATION_MIN_NS,
                                      NTSC_HSLEN_DURATION_TYP_NS,
                                      NTSC_HSLEN_DURATION_MAX_NS),
                          PARAM_RANGE(NTSC_HBP_DURATION_MIN_NS,
                                      NTSC_HBP_DURATION_TYP_NS,
                                      NTSC_HBP_DURATION_MAX_NS),
                          PARAM_RANGE(NTSC_HBLK_DURATION_MIN_NS,
                                      NTSC_HBLK_DURATION_TYP_NS,
                                      NTSC_HBLK_DURATION_MAX_NS),
                          16,
                          PARAM_FIELD(3, 3),
                          PARAM_FIELD(3, 3),
                          PARAM_FIELD(16, 17)),
        TV_MODE_PARAMETER(DRM_MODE_ANALOG_PAL,
                          PAL_LINES_NUMBER,
                          PAL_LINE_DURATION_NS,
                          PARAM_RANGE(PAL_HACT_DURATION_MIN_NS,
                                      PAL_HACT_DURATION_TYP_NS,
                                      PAL_HACT_DURATION_MAX_NS),
                          PARAM_RANGE(PAL_HFP_DURATION_MIN_NS,
                                      PAL_HFP_DURATION_TYP_NS,
                                      PAL_HFP_DURATION_MAX_NS),
                          PARAM_RANGE(PAL_HSLEN_DURATION_MIN_NS,
                                      PAL_HSLEN_DURATION_TYP_NS,
                                      PAL_HSLEN_DURATION_MAX_NS),
                          PARAM_RANGE(PAL_HBP_DURATION_MIN_NS,
                                      PAL_HBP_DURATION_TYP_NS,
                                      PAL_HBP_DURATION_MAX_NS),
                          PARAM_RANGE(PAL_HBLK_DURATION_MIN_NS,
                                      PAL_HBLK_DURATION_TYP_NS,
                                      PAL_HBLK_DURATION_MAX_NS),
                          12,

                          /*
                           * The front porch is actually 6 short sync
                           * pulses for the even field, and 5 for the
                           * odd field. Each sync takes half a life so
                           * the odd field front porch is shorter by
                           * half a line.
                           *
                           * In progressive, we're supposed to use 6
                           * pulses, so we're fine there
                           */
                          PARAM_FIELD(3, 2),

                          /*
                           * The vsync length is 5 long sync pulses,
                           * each field taking half a line. We're
                           * shorter for both fields by half a line.
                           *
                           * In progressive, we're supposed to use 5
                           * pulses, so we're off by half
                           * a line.
                           *
                           * In interlace, we're now off by half a line
                           * for the even field and one line for the odd
                           * field.
                           */
                          PARAM_FIELD(3, 3),

                          /*
                           * The back porch starts with post-equalizing
                           * pulses, consisting in 5 short sync pulses
                           * for the even field, 4 for the odd field. In
                           * progressive, it's 5 short syncs.
                           *
                           * In progressive, we thus have 2.5 lines,
                           * plus the 0.5 line we were missing
                           * previously, so we should use 3 lines.
                           *
                           * In interlace, the even field is in the
                           * exact same case than progressive. For the
                           * odd field, we should be using 2 lines but
                           * we're one line short, so we'll make up for
                           * it here by using 3.
                           *
                           * The entire blanking area is supposed to
                           * take 25 lines, so we also need to account
                           * for the rest of the blanking area that
                           * can't be in either the front porch or sync
                           * period.
                           */
                          PARAM_FIELD(19, 20)),
};

static int fill_analog_mode(struct drm_device *dev,
                            struct drm_display_mode *mode,
                            const struct analog_parameters *params,
                            unsigned long pixel_clock_hz,
                            unsigned int hactive,
                            unsigned int vactive,
                            bool interlace)
{
        unsigned long pixel_duration_ns = NSEC_PER_SEC / pixel_clock_hz;
        unsigned int htotal, vtotal;
        unsigned int max_hact, hact_duration_ns;
        unsigned int hblk, hblk_duration_ns;
        unsigned int hfp, hfp_duration_ns;
        unsigned int hslen, hslen_duration_ns;
        unsigned int hbp, hbp_duration_ns;
        unsigned int porches, porches_duration_ns;
        unsigned int vfp, vfp_min;
        unsigned int vbp, vbp_min;
        unsigned int vslen;
        bool bt601 = false;
        int porches_rem;
        u64 result;

        drm_dbg_kms(dev,
                    "Generating a %ux%u%c, %u-line mode with a %lu kHz clock\n",
                    hactive, vactive,
                    interlace ? 'i' : 'p',
                    params->num_lines,
                    pixel_clock_hz / 1000);

        max_hact = params->hact_ns.max / pixel_duration_ns;
        if (pixel_clock_hz == 13500000 && hactive > max_hact && hactive <= 720) {
                drm_dbg_kms(dev, "Trying to generate a BT.601 mode. Disabling checks.\n");
                bt601 = true;
        }

        /*
         * Our pixel duration is going to be round down by the division,
         * so rounding up is probably going to introduce even more
         * deviation.
         */
        result = (u64)params->line_duration_ns * pixel_clock_hz;
        do_div(result, NSEC_PER_SEC);
        htotal = result;

        drm_dbg_kms(dev, "Total Horizontal Number of Pixels: %u\n", htotal);

        hact_duration_ns = hactive * pixel_duration_ns;
        if (!bt601 &&
            (hact_duration_ns < params->hact_ns.min ||
             hact_duration_ns > params->hact_ns.max)) {
                drm_err(dev, "Invalid horizontal active area duration: %uns (min: %u, max %u)\n",
                        hact_duration_ns, params->hact_ns.min, params->hact_ns.max);
                return -EINVAL;
        }

        hblk = htotal - hactive;
        drm_dbg_kms(dev, "Horizontal Blanking Period: %u\n", hblk);

        hblk_duration_ns = hblk * pixel_duration_ns;
        if (!bt601 &&
            (hblk_duration_ns < params->hblk_ns.min ||
             hblk_duration_ns > params->hblk_ns.max)) {
                drm_err(dev, "Invalid horizontal blanking duration: %uns (min: %u, max %u)\n",
                        hblk_duration_ns, params->hblk_ns.min, params->hblk_ns.max);
                return -EINVAL;
        }

        hslen = DIV_ROUND_UP(params->hslen_ns.typ, pixel_duration_ns);
        drm_dbg_kms(dev, "Horizontal Sync Period: %u\n", hslen);

        hslen_duration_ns = hslen * pixel_duration_ns;
        if (!bt601 &&
            (hslen_duration_ns < params->hslen_ns.min ||
             hslen_duration_ns > params->hslen_ns.max)) {
                drm_err(dev, "Invalid horizontal sync duration: %uns (min: %u, max %u)\n",
                        hslen_duration_ns, params->hslen_ns.min, params->hslen_ns.max);
                return -EINVAL;
        }

        porches = hblk - hslen;
        drm_dbg_kms(dev, "Remaining horizontal pixels for both porches: %u\n", porches);

        porches_duration_ns = porches * pixel_duration_ns;
        if (!bt601 &&
            (porches_duration_ns > (params->hfp_ns.max + params->hbp_ns.max) ||
             porches_duration_ns < (params->hfp_ns.min + params->hbp_ns.min))) {
                drm_err(dev, "Invalid horizontal porches duration: %uns\n",
                        porches_duration_ns);
                return -EINVAL;
        }

        if (bt601) {
                hfp = params->bt601_hfp;
        } else {
                unsigned int hfp_min = DIV_ROUND_UP(params->hfp_ns.min,
                                                    pixel_duration_ns);
                unsigned int hbp_min = DIV_ROUND_UP(params->hbp_ns.min,
                                                    pixel_duration_ns);
                int porches_rem = porches - hfp_min - hbp_min;

                hfp = hfp_min + DIV_ROUND_UP(porches_rem, 2);
        }

        drm_dbg_kms(dev, "Horizontal Front Porch: %u\n", hfp);

        hfp_duration_ns = hfp * pixel_duration_ns;
        if (!bt601 &&
            (hfp_duration_ns < params->hfp_ns.min ||
             hfp_duration_ns > params->hfp_ns.max)) {
                drm_err(dev, "Invalid horizontal front porch duration: %uns (min: %u, max %u)\n",
                        hfp_duration_ns, params->hfp_ns.min, params->hfp_ns.max);
                return -EINVAL;
        }

        hbp = porches - hfp;
        drm_dbg_kms(dev, "Horizontal Back Porch: %u\n", hbp);

        hbp_duration_ns = hbp * pixel_duration_ns;
        if (!bt601 &&
            (hbp_duration_ns < params->hbp_ns.min ||
             hbp_duration_ns > params->hbp_ns.max)) {
                drm_err(dev, "Invalid horizontal back porch duration: %uns (min: %u, max %u)\n",
                        hbp_duration_ns, params->hbp_ns.min, params->hbp_ns.max);
                return -EINVAL;
        }

        if (htotal != (hactive + hfp + hslen + hbp))
                return -EINVAL;

        mode->clock = pixel_clock_hz / 1000;
        mode->hdisplay = hactive;
        mode->hsync_start = mode->hdisplay + hfp;
        mode->hsync_end = mode->hsync_start + hslen;
        mode->htotal = mode->hsync_end + hbp;

        if (interlace) {
                vfp_min = params->vfp_lines.even + params->vfp_lines.odd;
                vbp_min = params->vbp_lines.even + params->vbp_lines.odd;
                vslen = params->vslen_lines.even + params->vslen_lines.odd;
        } else {
                /*
                 * By convention, NTSC (aka 525/60) systems start with
                 * the even field, but PAL (aka 625/50) systems start
                 * with the odd one.
                 *
                 * PAL systems also have asymmetric timings between the
                 * even and odd field, while NTSC is symmetric.
                 *
                 * Moreover, if we want to create a progressive mode for
                 * PAL, we need to use the odd field timings.
                 *
                 * Since odd == even for NTSC, we can just use the odd
                 * one all the time to simplify the code a bit.
                 */
                vfp_min = params->vfp_lines.odd;
                vbp_min = params->vbp_lines.odd;
                vslen = params->vslen_lines.odd;
        }

        drm_dbg_kms(dev, "Vertical Sync Period: %u\n", vslen);

        porches = params->num_lines - vactive - vslen;
        drm_dbg_kms(dev, "Remaining vertical pixels for both porches: %u\n", porches);

        porches_rem = porches - vfp_min - vbp_min;
        vfp = vfp_min + (porches_rem / 2);
        drm_dbg_kms(dev, "Vertical Front Porch: %u\n", vfp);

        vbp = porches - vfp;
        drm_dbg_kms(dev, "Vertical Back Porch: %u\n", vbp);

        vtotal = vactive + vfp + vslen + vbp;
        if (params->num_lines != vtotal) {
                drm_err(dev, "Invalid vertical total: %upx (expected %upx)\n",
                        vtotal, params->num_lines);
                return -EINVAL;
        }

        mode->vdisplay = vactive;
        mode->vsync_start = mode->vdisplay + vfp;
        mode->vsync_end = mode->vsync_start + vslen;
        mode->vtotal = mode->vsync_end + vbp;

        if (mode->vtotal != params->num_lines)
                return -EINVAL;

        mode->type = DRM_MODE_TYPE_DRIVER;
        mode->flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC;
        if (interlace)
                mode->flags |= DRM_MODE_FLAG_INTERLACE;

        drm_mode_set_name(mode);

        drm_dbg_kms(dev, "Generated mode " DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));

        return 0;
}

/**
 * drm_analog_tv_mode - create a display mode for an analog TV
 * @dev: drm device
 * @tv_mode: TV Mode standard to create a mode for. See DRM_MODE_TV_MODE_*.
 * @pixel_clock_hz: Pixel Clock Frequency, in Hertz
 * @hdisplay: hdisplay size
 * @vdisplay: vdisplay size
 * @interlace: whether to compute an interlaced mode
 *
 * This function creates a struct drm_display_mode instance suited for
 * an analog TV output, for one of the usual analog TV modes. Where
 * this is DRM_MODE_TV_MODE_MONOCHROME, a 625-line mode will be created.
 *
 * Note that @hdisplay is larger than the usual constraints for the PAL
 * and NTSC timings, and we'll choose to ignore most timings constraints
 * to reach those resolutions.
 *
 * Returns:
 * A pointer to the mode, allocated with drm_mode_create(). Returns NULL
 * on error.
 */
struct drm_display_mode *drm_analog_tv_mode(struct drm_device *dev,
                                            enum drm_connector_tv_mode tv_mode,
                                            unsigned long pixel_clock_hz,
                                            unsigned int hdisplay,
                                            unsigned int vdisplay,
                                            bool interlace)
{
        struct drm_display_mode *mode;
        enum drm_mode_analog analog;
        int ret;

        switch (tv_mode) {
        case DRM_MODE_TV_MODE_NTSC:
                fallthrough;
        case DRM_MODE_TV_MODE_NTSC_443:
                fallthrough;
        case DRM_MODE_TV_MODE_NTSC_J:
                fallthrough;
        case DRM_MODE_TV_MODE_PAL_M:
                analog = DRM_MODE_ANALOG_NTSC;
                break;

        case DRM_MODE_TV_MODE_PAL:
                fallthrough;
        case DRM_MODE_TV_MODE_PAL_N:
                fallthrough;
        case DRM_MODE_TV_MODE_SECAM:
                fallthrough;
        case DRM_MODE_TV_MODE_MONOCHROME:
                analog = DRM_MODE_ANALOG_PAL;
                break;

        default:
                return NULL;
        }

        mode = drm_mode_create(dev);
        if (!mode)
                return NULL;

        ret = fill_analog_mode(dev, mode,
                               &tv_modes_parameters[analog],
                               pixel_clock_hz, hdisplay, vdisplay, interlace);
        if (ret)
                goto err_free_mode;

        return mode;

err_free_mode:
        drm_mode_destroy(dev, mode);
        return NULL;
}
EXPORT_SYMBOL(drm_analog_tv_mode);

/**
 * drm_cvt_mode -create a modeline based on the CVT algorithm
 * @dev: drm device
 * @hdisplay: hdisplay size
 * @vdisplay: vdisplay size
 * @vrefresh: vrefresh rate
 * @reduced: whether to use reduced blanking
 * @interlaced: whether to compute an interlaced mode
 * @margins: whether to add margins (borders)
 *
 * This function is called to generate the modeline based on CVT algorithm
 * according to the hdisplay, vdisplay, vrefresh.
 * It is based from the VESA(TM) Coordinated Video Timing Generator by
 * Graham Loveridge April 9, 2003 available at
 * http://www.elo.utfsm.cl/~elo212/docs/CVTd6r1.xls
 *
 * And it is copied from xf86CVTmode in xserver/hw/xfree86/modes/xf86cvt.c.
 * What I have done is to translate it by using integer calculation.
 *
 * Returns:
 * The modeline based on the CVT algorithm stored in a drm_display_mode object.
 * The display mode object is allocated with drm_mode_create(). Returns NULL
 * when no mode could be allocated.
 */
struct drm_display_mode *drm_cvt_mode(struct drm_device *dev, int hdisplay,
                                      int vdisplay, int vrefresh,
                                      bool reduced, bool interlaced, bool margins)
{
#define HV_FACTOR                        1000
        /* 1) top/bottom margin size (% of height) - default: 1.8, */
#define        CVT_MARGIN_PERCENTAGE                18
        /* 2) character cell horizontal granularity (pixels) - default 8 */
#define        CVT_H_GRANULARITY                8
        /* 3) Minimum vertical porch (lines) - default 3 */
#define        CVT_MIN_V_PORCH                        3
        /* 4) Minimum number of vertical back porch lines - default 6 */
#define        CVT_MIN_V_BPORCH                6
        /* Pixel Clock step (kHz) */
#define CVT_CLOCK_STEP                        250
        struct drm_display_mode *drm_mode;
        unsigned int vfieldrate, hperiod;
        int hdisplay_rnd, hmargin, vdisplay_rnd, vmargin, vsync;
        int interlace;
        u64 tmp;

        if (!hdisplay || !vdisplay)
                return NULL;

        /* allocate the drm_display_mode structure. If failure, we will
         * return directly
         */
        drm_mode = drm_mode_create(dev);
        if (!drm_mode)
                return NULL;

        /* the CVT default refresh rate is 60Hz */
        if (!vrefresh)
                vrefresh = 60;

        /* the required field fresh rate */
        if (interlaced)
                vfieldrate = vrefresh * 2;
        else
                vfieldrate = vrefresh;

        /* horizontal pixels */
        hdisplay_rnd = hdisplay - (hdisplay % CVT_H_GRANULARITY);

        /* determine the left&right borders */
        hmargin = 0;
        if (margins) {
                hmargin = hdisplay_rnd * CVT_MARGIN_PERCENTAGE / 1000;
                hmargin -= hmargin % CVT_H_GRANULARITY;
        }
        /* find the total active pixels */
        drm_mode->hdisplay = hdisplay_rnd + 2 * hmargin;

        /* find the number of lines per field */
        if (interlaced)
                vdisplay_rnd = vdisplay / 2;
        else
                vdisplay_rnd = vdisplay;

        /* find the top & bottom borders */
        vmargin = 0;
        if (margins)
                vmargin = vdisplay_rnd * CVT_MARGIN_PERCENTAGE / 1000;

        drm_mode->vdisplay = vdisplay + 2 * vmargin;

        /* Interlaced */
        if (interlaced)
                interlace = 1;
        else
                interlace = 0;

        /* Determine VSync Width from aspect ratio */
        if (!(vdisplay % 3) && ((vdisplay * 4 / 3) == hdisplay))
                vsync = 4;
        else if (!(vdisplay % 9) && ((vdisplay * 16 / 9) == hdisplay))
                vsync = 5;
        else if (!(vdisplay % 10) && ((vdisplay * 16 / 10) == hdisplay))
                vsync = 6;
        else if (!(vdisplay % 4) && ((vdisplay * 5 / 4) == hdisplay))
                vsync = 7;
        else if (!(vdisplay % 9) && ((vdisplay * 15 / 9) == hdisplay))
                vsync = 7;
        else /* custom */
                vsync = 10;

        if (!reduced) {
                /* simplify the GTF calculation */
                /* 4) Minimum time of vertical sync + back porch interval (µs)
                 * default 550.0
                 */
                int tmp1, tmp2;
#define CVT_MIN_VSYNC_BP        550
                /* 3) Nominal HSync width (% of line period) - default 8 */
#define CVT_HSYNC_PERCENTAGE        8
                unsigned int hblank_percentage;
                int vsyncandback_porch, __maybe_unused vback_porch, hblank;

                /* estimated the horizontal period */
                tmp1 = HV_FACTOR * 1000000  -
                                CVT_MIN_VSYNC_BP * HV_FACTOR * vfieldrate;
                tmp2 = (vdisplay_rnd + 2 * vmargin + CVT_MIN_V_PORCH) * 2 +
                                interlace;
                hperiod = tmp1 * 2 / (tmp2 * vfieldrate);

                tmp1 = CVT_MIN_VSYNC_BP * HV_FACTOR / hperiod + 1;
                /* 9. Find number of lines in sync + backporch */
                if (tmp1 < (vsync + CVT_MIN_V_PORCH))
                        vsyncandback_porch = vsync + CVT_MIN_V_PORCH;
                else
                        vsyncandback_porch = tmp1;
                /* 10. Find number of lines in back porch */
                vback_porch = vsyncandback_porch - vsync;
                drm_mode->vtotal = vdisplay_rnd + 2 * vmargin +
                                vsyncandback_porch + CVT_MIN_V_PORCH;
                /* 5) Definition of Horizontal blanking time limitation */
                /* Gradient (%/kHz) - default 600 */
#define CVT_M_FACTOR        600
                /* Offset (%) - default 40 */
#define CVT_C_FACTOR        40
                /* Blanking time scaling factor - default 128 */
#define CVT_K_FACTOR        128
                /* Scaling factor weighting - default 20 */
#define CVT_J_FACTOR        20
#define CVT_M_PRIME        (CVT_M_FACTOR * CVT_K_FACTOR / 256)
#define CVT_C_PRIME        ((CVT_C_FACTOR - CVT_J_FACTOR) * CVT_K_FACTOR / 256 + \
                         CVT_J_FACTOR)
                /* 12. Find ideal blanking duty cycle from formula */
                hblank_percentage = CVT_C_PRIME * HV_FACTOR - CVT_M_PRIME *
                                        hperiod / 1000;
                /* 13. Blanking time */
                if (hblank_percentage < 20 * HV_FACTOR)
                        hblank_percentage = 20 * HV_FACTOR;
                hblank = drm_mode->hdisplay * hblank_percentage /
                         (100 * HV_FACTOR - hblank_percentage);
                hblank -= hblank % (2 * CVT_H_GRANULARITY);
                /* 14. find the total pixels per line */
                drm_mode->htotal = drm_mode->hdisplay + hblank;
                drm_mode->hsync_end = drm_mode->hdisplay + hblank / 2;
                drm_mode->hsync_start = drm_mode->hsync_end -
                        (drm_mode->htotal * CVT_HSYNC_PERCENTAGE) / 100;
                drm_mode->hsync_start += CVT_H_GRANULARITY -
                        drm_mode->hsync_start % CVT_H_GRANULARITY;
                /* fill the Vsync values */
                drm_mode->vsync_start = drm_mode->vdisplay + CVT_MIN_V_PORCH;
                drm_mode->vsync_end = drm_mode->vsync_start + vsync;
        } else {
                /* Reduced blanking */
                /* Minimum vertical blanking interval time (µs)- default 460 */
#define CVT_RB_MIN_VBLANK        460
                /* Fixed number of clocks for horizontal sync */
#define CVT_RB_H_SYNC                32
                /* Fixed number of clocks for horizontal blanking */
#define CVT_RB_H_BLANK                160
                /* Fixed number of lines for vertical front porch - default 3*/
#define CVT_RB_VFPORCH                3
                int vbilines;
                int tmp1, tmp2;
                /* 8. Estimate Horizontal period. */
                tmp1 = HV_FACTOR * 1000000 -
                        CVT_RB_MIN_VBLANK * HV_FACTOR * vfieldrate;
                tmp2 = vdisplay_rnd + 2 * vmargin;
                hperiod = tmp1 / (tmp2 * vfieldrate);
                /* 9. Find number of lines in vertical blanking */
                vbilines = CVT_RB_MIN_VBLANK * HV_FACTOR / hperiod + 1;
                /* 10. Check if vertical blanking is sufficient */
                if (vbilines < (CVT_RB_VFPORCH + vsync + CVT_MIN_V_BPORCH))
                        vbilines = CVT_RB_VFPORCH + vsync + CVT_MIN_V_BPORCH;
                /* 11. Find total number of lines in vertical field */
                drm_mode->vtotal = vdisplay_rnd + 2 * vmargin + vbilines;
                /* 12. Find total number of pixels in a line */
                drm_mode->htotal = drm_mode->hdisplay + CVT_RB_H_BLANK;
                /* Fill in HSync values */
                drm_mode->hsync_end = drm_mode->hdisplay + CVT_RB_H_BLANK / 2;
                drm_mode->hsync_start = drm_mode->hsync_end - CVT_RB_H_SYNC;
                /* Fill in VSync values */
                drm_mode->vsync_start = drm_mode->vdisplay + CVT_RB_VFPORCH;
                drm_mode->vsync_end = drm_mode->vsync_start + vsync;
        }
        /* 15/13. Find pixel clock frequency (kHz for xf86) */
        tmp = drm_mode->htotal; /* perform intermediate calcs in u64 */
        tmp *= HV_FACTOR * 1000;
        do_div(tmp, hperiod);
        tmp -= drm_mode->clock % CVT_CLOCK_STEP;
        drm_mode->clock = tmp;
        /* 18/16. Find actual vertical frame frequency */
        /* ignore - just set the mode flag for interlaced */
        if (interlaced) {
                drm_mode->vtotal *= 2;
                drm_mode->flags |= DRM_MODE_FLAG_INTERLACE;
        }
        /* Fill the mode line name */
        drm_mode_set_name(drm_mode);
        if (reduced)
                drm_mode->flags |= (DRM_MODE_FLAG_PHSYNC |
                                        DRM_MODE_FLAG_NVSYNC);
        else
                drm_mode->flags |= (DRM_MODE_FLAG_PVSYNC |
                                        DRM_MODE_FLAG_NHSYNC);

        return drm_mode;
}
EXPORT_SYMBOL(drm_cvt_mode);

/**
 * drm_gtf_mode_complex - create the modeline based on the full GTF algorithm
 * @dev: drm device
 * @hdisplay: hdisplay size
 * @vdisplay: vdisplay size
 * @vrefresh: vrefresh rate.
 * @interlaced: whether to compute an interlaced mode
 * @margins: desired margin (borders) size
 * @GTF_M: extended GTF formula parameters
 * @GTF_2C: extended GTF formula parameters
 * @GTF_K: extended GTF formula parameters
 * @GTF_2J: extended GTF formula parameters
 *
 * GTF feature blocks specify C and J in multiples of 0.5, so we pass them
 * in here multiplied by two.  For a C of 40, pass in 80.
 *
 * Returns:
 * The modeline based on the full GTF algorithm stored in a drm_display_mode object.
 * The display mode object is allocated with drm_mode_create(). Returns NULL
 * when no mode could be allocated.
 */
struct drm_display_mode *
drm_gtf_mode_complex(struct drm_device *dev, int hdisplay, int vdisplay,
                     int vrefresh, bool interlaced, int margins,
                     int GTF_M, int GTF_2C, int GTF_K, int GTF_2J)
{        /* 1) top/bottom margin size (% of height) - default: 1.8, */
#define        GTF_MARGIN_PERCENTAGE                18
        /* 2) character cell horizontal granularity (pixels) - default 8 */
#define        GTF_CELL_GRAN                        8
        /* 3) Minimum vertical porch (lines) - default 3 */
#define        GTF_MIN_V_PORCH                        1
        /* width of vsync in lines */
#define V_SYNC_RQD                        3
        /* width of hsync as % of total line */
#define H_SYNC_PERCENT                        8
        /* min time of vsync + back porch (microsec) */
#define MIN_VSYNC_PLUS_BP                550
        /* C' and M' are part of the Blanking Duty Cycle computation */
#define GTF_C_PRIME        ((((GTF_2C - GTF_2J) * GTF_K / 256) + GTF_2J) / 2)
#define GTF_M_PRIME        (GTF_K * GTF_M / 256)
        struct drm_display_mode *drm_mode;
        unsigned int hdisplay_rnd, vdisplay_rnd, vfieldrate_rqd;
        int top_margin, bottom_margin;
        int interlace;
        unsigned int hfreq_est;
        int vsync_plus_bp, __maybe_unused vback_porch;
        unsigned int vtotal_lines, __maybe_unused vfieldrate_est;
        unsigned int __maybe_unused hperiod;
        unsigned int vfield_rate, __maybe_unused vframe_rate;
        int left_margin, right_margin;
        unsigned int total_active_pixels, ideal_duty_cycle;
        unsigned int hblank, total_pixels, pixel_freq;
        int hsync, hfront_porch, vodd_front_porch_lines;
        unsigned int tmp1, tmp2;

        if (!hdisplay || !vdisplay)
                return NULL;

        drm_mode = drm_mode_create(dev);
        if (!drm_mode)
                return NULL;

        /* 1. In order to give correct results, the number of horizontal
         * pixels requested is first processed to ensure that it is divisible
         * by the character size, by rounding it to the nearest character
         * cell boundary:
         */
        hdisplay_rnd = (hdisplay + GTF_CELL_GRAN / 2) / GTF_CELL_GRAN;
        hdisplay_rnd = hdisplay_rnd * GTF_CELL_GRAN;

        /* 2. If interlace is requested, the number of vertical lines assumed
         * by the calculation must be halved, as the computation calculates
         * the number of vertical lines per field.
         */
        if (interlaced)
                vdisplay_rnd = vdisplay / 2;
        else
                vdisplay_rnd = vdisplay;

        /* 3. Find the frame rate required: */
        if (interlaced)
                vfieldrate_rqd = vrefresh * 2;
        else
                vfieldrate_rqd = vrefresh;

        /* 4. Find number of lines in Top margin: */
        top_margin = 0;
        if (margins)
                top_margin = (vdisplay_rnd * GTF_MARGIN_PERCENTAGE + 500) /
                                1000;
        /* 5. Find number of lines in bottom margin: */
        bottom_margin = top_margin;

        /* 6. If interlace is required, then set variable interlace: */
        if (interlaced)
                interlace = 1;
        else
                interlace = 0;

        /* 7. Estimate the Horizontal frequency */
        {
                tmp1 = (1000000  - MIN_VSYNC_PLUS_BP * vfieldrate_rqd) / 500;
                tmp2 = (vdisplay_rnd + 2 * top_margin + GTF_MIN_V_PORCH) *
                                2 + interlace;
                hfreq_est = (tmp2 * 1000 * vfieldrate_rqd) / tmp1;
        }

        /* 8. Find the number of lines in V sync + back porch */
        /* [V SYNC+BP] = RINT(([MIN VSYNC+BP] * hfreq_est / 1000000)) */
        vsync_plus_bp = MIN_VSYNC_PLUS_BP * hfreq_est / 1000;
        vsync_plus_bp = (vsync_plus_bp + 500) / 1000;
        /*  9. Find the number of lines in V back porch alone: */
        vback_porch = vsync_plus_bp - V_SYNC_RQD;
        /*  10. Find the total number of lines in Vertical field period: */
        vtotal_lines = vdisplay_rnd + top_margin + bottom_margin +
                        vsync_plus_bp + GTF_MIN_V_PORCH;
        /*  11. Estimate the Vertical field frequency: */
        vfieldrate_est = hfreq_est / vtotal_lines;
        /*  12. Find the actual horizontal period: */
        hperiod = 1000000 / (vfieldrate_rqd * vtotal_lines);

        /*  13. Find the actual Vertical field frequency: */
        vfield_rate = hfreq_est / vtotal_lines;
        /*  14. Find the Vertical frame frequency: */
        if (interlaced)
                vframe_rate = vfield_rate / 2;
        else
                vframe_rate = vfield_rate;
        /*  15. Find number of pixels in left margin: */
        if (margins)
                left_margin = (hdisplay_rnd * GTF_MARGIN_PERCENTAGE + 500) /
                                1000;
        else
                left_margin = 0;

        /* 16.Find number of pixels in right margin: */
        right_margin = left_margin;
        /* 17.Find total number of active pixels in image and left and right */
        total_active_pixels = hdisplay_rnd + left_margin + right_margin;
        /* 18.Find the ideal blanking duty cycle from blanking duty cycle */
        ideal_duty_cycle = GTF_C_PRIME * 1000 -
                                (GTF_M_PRIME * 1000000 / hfreq_est);
        /* 19.Find the number of pixels in the blanking time to the nearest
         * double character cell: */
        hblank = total_active_pixels * ideal_duty_cycle /
                        (100000 - ideal_duty_cycle);
        hblank = (hblank + GTF_CELL_GRAN) / (2 * GTF_CELL_GRAN);
        hblank = hblank * 2 * GTF_CELL_GRAN;
        /* 20.Find total number of pixels: */
        total_pixels = total_active_pixels + hblank;
        /* 21.Find pixel clock frequency: */
        pixel_freq = total_pixels * hfreq_est / 1000;
        /* Stage 1 computations are now complete; I should really pass
         * the results to another function and do the Stage 2 computations,
         * but I only need a few more values so I'll just append the
         * computations here for now */
        /* 17. Find the number of pixels in the horizontal sync period: */
        hsync = H_SYNC_PERCENT * total_pixels / 100;
        hsync = (hsync + GTF_CELL_GRAN / 2) / GTF_CELL_GRAN;
        hsync = hsync * GTF_CELL_GRAN;
        /* 18. Find the number of pixels in horizontal front porch period */
        hfront_porch = hblank / 2 - hsync;
        /*  36. Find the number of lines in the odd front porch period: */
        vodd_front_porch_lines = GTF_MIN_V_PORCH ;

        /* finally, pack the results in the mode struct */
        drm_mode->hdisplay = hdisplay_rnd;
        drm_mode->hsync_start = hdisplay_rnd + hfront_porch;
        drm_mode->hsync_end = drm_mode->hsync_start + hsync;
        drm_mode->htotal = total_pixels;
        drm_mode->vdisplay = vdisplay_rnd;
        drm_mode->vsync_start = vdisplay_rnd + vodd_front_porch_lines;
        drm_mode->vsync_end = drm_mode->vsync_start + V_SYNC_RQD;
        drm_mode->vtotal = vtotal_lines;

        drm_mode->clock = pixel_freq;

        if (interlaced) {
                drm_mode->vtotal *= 2;
                drm_mode->flags |= DRM_MODE_FLAG_INTERLACE;
        }

        drm_mode_set_name(drm_mode);
        if (GTF_M == 600 && GTF_2C == 80 && GTF_K == 128 && GTF_2J == 40)
                drm_mode->flags = DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC;
        else
                drm_mode->flags = DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC;

        return drm_mode;
}
EXPORT_SYMBOL(drm_gtf_mode_complex);

/**
 * drm_gtf_mode - create the modeline based on the GTF algorithm
 * @dev: drm device
 * @hdisplay: hdisplay size
 * @vdisplay: vdisplay size
 * @vrefresh: vrefresh rate.
 * @interlaced: whether to compute an interlaced mode
 * @margins: desired margin (borders) size
 *
 * return the modeline based on GTF algorithm
 *
 * This function is to create the modeline based on the GTF algorithm.
 * Generalized Timing Formula is derived from:
 *
 *        GTF Spreadsheet by Andy Morrish (1/5/97)
 *        available at https://www.vesa.org
 *
 * And it is copied from the file of xserver/hw/xfree86/modes/xf86gtf.c.
 * What I have done is to translate it by using integer calculation.
 * I also refer to the function of fb_get_mode in the file of
 * drivers/video/fbmon.c
 *
 * Standard GTF parameters::
 *
 *     M = 600
 *     C = 40
 *     K = 128
 *     J = 20
 *
 * Returns:
 * The modeline based on the GTF algorithm stored in a drm_display_mode object.
 * The display mode object is allocated with drm_mode_create(). Returns NULL
 * when no mode could be allocated.
 */
struct drm_display_mode *
drm_gtf_mode(struct drm_device *dev, int hdisplay, int vdisplay, int vrefresh,
             bool interlaced, int margins)
{
        return drm_gtf_mode_complex(dev, hdisplay, vdisplay, vrefresh,
                                    interlaced, margins,
                                    600, 40 * 2, 128, 20 * 2);
}
EXPORT_SYMBOL(drm_gtf_mode);

#ifdef CONFIG_VIDEOMODE_HELPERS
/**
 * drm_display_mode_from_videomode - fill in @dmode using @vm,
 * @vm: videomode structure to use as source
 * @dmode: drm_display_mode structure to use as destination
 *
 * Fills out @dmode using the display mode specified in @vm.
 */
void drm_display_mode_from_videomode(const struct videomode *vm,
                                     struct drm_display_mode *dmode)
{
        dmode->hdisplay = vm->hactive;
        dmode->hsync_start = dmode->hdisplay + vm->hfront_porch;
        dmode->hsync_end = dmode->hsync_start + vm->hsync_len;
        dmode->htotal = dmode->hsync_end + vm->hback_porch;

        dmode->vdisplay = vm->vactive;
        dmode->vsync_start = dmode->vdisplay + vm->vfront_porch;
        dmode->vsync_end = dmode->vsync_start + vm->vsync_len;
        dmode->vtotal = dmode->vsync_end + vm->vback_porch;

        dmode->clock = vm->pixelclock / 1000;

        dmode->flags = 0;
        if (vm->flags & DISPLAY_FLAGS_HSYNC_HIGH)
                dmode->flags |= DRM_MODE_FLAG_PHSYNC;
        else if (vm->flags & DISPLAY_FLAGS_HSYNC_LOW)
                dmode->flags |= DRM_MODE_FLAG_NHSYNC;
        if (vm->flags & DISPLAY_FLAGS_VSYNC_HIGH)
                dmode->flags |= DRM_MODE_FLAG_PVSYNC;
        else if (vm->flags & DISPLAY_FLAGS_VSYNC_LOW)
                dmode->flags |= DRM_MODE_FLAG_NVSYNC;
        if (vm->flags & DISPLAY_FLAGS_INTERLACED)
                dmode->flags |= DRM_MODE_FLAG_INTERLACE;
        if (vm->flags & DISPLAY_FLAGS_DOUBLESCAN)
                dmode->flags |= DRM_MODE_FLAG_DBLSCAN;
        if (vm->flags & DISPLAY_FLAGS_DOUBLECLK)
                dmode->flags |= DRM_MODE_FLAG_DBLCLK;
        drm_mode_set_name(dmode);
}
EXPORT_SYMBOL_GPL(drm_display_mode_from_videomode);

/**
 * drm_display_mode_to_videomode - fill in @vm using @dmode,
 * @dmode: drm_display_mode structure to use as source
 * @vm: videomode structure to use as destination
 *
 * Fills out @vm using the display mode specified in @dmode.
 */
void drm_display_mode_to_videomode(const struct drm_display_mode *dmode,
                                   struct videomode *vm)
{
        vm->hactive = dmode->hdisplay;
        vm->hfront_porch = dmode->hsync_start - dmode->hdisplay;
        vm->hsync_len = dmode->hsync_end - dmode->hsync_start;
        vm->hback_porch = dmode->htotal - dmode->hsync_end;

        vm->vactive = dmode->vdisplay;
        vm->vfront_porch = dmode->vsync_start - dmode->vdisplay;
        vm->vsync_len = dmode->vsync_end - dmode->vsync_start;
        vm->vback_porch = dmode->vtotal - dmode->vsync_end;

        vm->pixelclock = dmode->clock * 1000;

        vm->flags = 0;
        if (dmode->flags & DRM_MODE_FLAG_PHSYNC)
                vm->flags |= DISPLAY_FLAGS_HSYNC_HIGH;
        else if (dmode->flags & DRM_MODE_FLAG_NHSYNC)
                vm->flags |= DISPLAY_FLAGS_HSYNC_LOW;
        if (dmode->flags & DRM_MODE_FLAG_PVSYNC)
                vm->flags |= DISPLAY_FLAGS_VSYNC_HIGH;
        else if (dmode->flags & DRM_MODE_FLAG_NVSYNC)
                vm->flags |= DISPLAY_FLAGS_VSYNC_LOW;
        if (dmode->flags & DRM_MODE_FLAG_INTERLACE)
                vm->flags |= DISPLAY_FLAGS_INTERLACED;
        if (dmode->flags & DRM_MODE_FLAG_DBLSCAN)
                vm->flags |= DISPLAY_FLAGS_DOUBLESCAN;
        if (dmode->flags & DRM_MODE_FLAG_DBLCLK)
                vm->flags |= DISPLAY_FLAGS_DOUBLECLK;
}
EXPORT_SYMBOL_GPL(drm_display_mode_to_videomode);

/**
 * drm_bus_flags_from_videomode - extract information about pixelclk and
 * DE polarity from videomode and store it in a separate variable
 * @vm: videomode structure to use
 * @bus_flags: information about pixelclk, sync and DE polarity will be stored
 * here
 *
 * Sets DRM_BUS_FLAG_DE_(LOW|HIGH),  DRM_BUS_FLAG_PIXDATA_DRIVE_(POS|NEG)EDGE
 * and DISPLAY_FLAGS_SYNC_(POS|NEG)EDGE in @bus_flags according to DISPLAY_FLAGS
 * found in @vm
 */
void drm_bus_flags_from_videomode(const struct videomode *vm, u32 *bus_flags)
{
        *bus_flags = 0;
        if (vm->flags & DISPLAY_FLAGS_PIXDATA_POSEDGE)
                *bus_flags |= DRM_BUS_FLAG_PIXDATA_DRIVE_POSEDGE;
        if (vm->flags & DISPLAY_FLAGS_PIXDATA_NEGEDGE)
                *bus_flags |= DRM_BUS_FLAG_PIXDATA_DRIVE_NEGEDGE;

        if (vm->flags & DISPLAY_FLAGS_SYNC_POSEDGE)
                *bus_flags |= DRM_BUS_FLAG_SYNC_DRIVE_POSEDGE;
        if (vm->flags & DISPLAY_FLAGS_SYNC_NEGEDGE)
                *bus_flags |= DRM_BUS_FLAG_SYNC_DRIVE_NEGEDGE;

        if (vm->flags & DISPLAY_FLAGS_DE_LOW)
                *bus_flags |= DRM_BUS_FLAG_DE_LOW;
        if (vm->flags & DISPLAY_FLAGS_DE_HIGH)
                *bus_flags |= DRM_BUS_FLAG_DE_HIGH;
}
EXPORT_SYMBOL_GPL(drm_bus_flags_from_videomode);

#ifdef CONFIG_OF
/**
 * of_get_drm_display_mode - get a drm_display_mode from devicetree
 * @np: device_node with the timing specification
 * @dmode: will be set to the return value
 * @bus_flags: information about pixelclk, sync and DE polarity
 * @index: index into the list of display timings in devicetree
 *
 * This function is expensive and should only be used, if only one mode is to be
 * read from DT. To get multiple modes start with of_get_display_timings and
 * work with that instead.
 *
 * Returns:
 * 0 on success, a negative errno code when no of videomode node was found.
 */
int of_get_drm_display_mode(struct device_node *np,
                            struct drm_display_mode *dmode, u32 *bus_flags,
                            int index)
{
        struct videomode vm;
        int ret;

        ret = of_get_videomode(np, &vm, index);
        if (ret)
                return ret;

        drm_display_mode_from_videomode(&vm, dmode);
        if (bus_flags)
                drm_bus_flags_from_videomode(&vm, bus_flags);

        pr_debug("%pOF: got %dx%d display mode: " DRM_MODE_FMT "\n",
                 np, vm.hactive, vm.vactive, DRM_MODE_ARG(dmode));

        return 0;
}
EXPORT_SYMBOL_GPL(of_get_drm_display_mode);

/**
 * of_get_drm_panel_display_mode - get a panel-timing drm_display_mode from devicetree
 * @np: device_node with the panel-timing specification
 * @dmode: will be set to the return value
 * @bus_flags: information about pixelclk, sync and DE polarity
 *
 * The mandatory Device Tree properties width-mm and height-mm
 * are read and set on the display mode.
 *
 * Returns:
 * Zero on success, negative error code on failure.
 */
int of_get_drm_panel_display_mode(struct device_node *np,
                                  struct drm_display_mode *dmode, u32 *bus_flags)
{
        u32 width_mm = 0, height_mm = 0;
        struct display_timing timing;
        struct videomode vm;
        int ret;

        ret = of_get_display_timing(np, "panel-timing", &timing);
        if (ret)
                return ret;

        videomode_from_timing(&timing, &vm);

        memset(dmode, 0, sizeof(*dmode));
        drm_display_mode_from_videomode(&vm, dmode);
        if (bus_flags)
                drm_bus_flags_from_videomode(&vm, bus_flags);

        ret = of_property_read_u32(np, "width-mm", &width_mm);
        if (ret)
                return ret;

        ret = of_property_read_u32(np, "height-mm", &height_mm);
        if (ret)
                return ret;

        dmode->width_mm = width_mm;
        dmode->height_mm = height_mm;

        pr_debug(DRM_MODE_FMT "\n", DRM_MODE_ARG(dmode));

        return 0;
}
EXPORT_SYMBOL_GPL(of_get_drm_panel_display_mode);
#endif /* CONFIG_OF */
#endif /* CONFIG_VIDEOMODE_HELPERS */

/**
 * drm_mode_set_name - set the name on a mode
 * @mode: name will be set in this mode
 *
 * Set the name of @mode to a standard format which is <hdisplay>x<vdisplay>
 * with an optional 'i' suffix for interlaced modes.
 */
void drm_mode_set_name(struct drm_display_mode *mode)
{
        bool interlaced = !!(mode->flags & DRM_MODE_FLAG_INTERLACE);

        snprintf(mode->name, DRM_DISPLAY_MODE_LEN, "%dx%d%s",
                 mode->hdisplay, mode->vdisplay,
                 interlaced ? "i" : "");
}
EXPORT_SYMBOL(drm_mode_set_name);

/**
 * drm_mode_vrefresh - get the vrefresh of a mode
 * @mode: mode
 *
 * Returns:
 * @modes's vrefresh rate in Hz, rounded to the nearest integer. Calculates the
 * value first if it is not yet set.
 */
int drm_mode_vrefresh(const struct drm_display_mode *mode)
{
        unsigned int num, den;

        if (mode->htotal == 0 || mode->vtotal == 0)
                return 0;

        num = mode->clock;
        den = mode->htotal * mode->vtotal;

        if (mode->flags & DRM_MODE_FLAG_INTERLACE)
                num *= 2;
        if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
                den *= 2;
        if (mode->vscan > 1)
                den *= mode->vscan;

        return DIV_ROUND_CLOSEST_ULL(mul_u32_u32(num, 1000), den);
}
EXPORT_SYMBOL(drm_mode_vrefresh);

/**
 * drm_mode_get_hv_timing - Fetches hdisplay/vdisplay for given mode
 * @mode: mode to query
 * @hdisplay: hdisplay value to fill in
 * @vdisplay: vdisplay value to fill in
 *
 * The vdisplay value will be doubled if the specified mode is a stereo mode of
 * the appropriate layout.
 */
void drm_mode_get_hv_timing(const struct drm_display_mode *mode,
                            int *hdisplay, int *vdisplay)
{
        struct drm_display_mode adjusted;

        drm_mode_init(&adjusted, mode);

        drm_mode_set_crtcinfo(&adjusted, CRTC_STEREO_DOUBLE_ONLY);
        *hdisplay = adjusted.crtc_hdisplay;
        *vdisplay = adjusted.crtc_vdisplay;
}
EXPORT_SYMBOL(drm_mode_get_hv_timing);

/**
 * drm_mode_set_crtcinfo - set CRTC modesetting timing parameters
 * @p: mode
 * @adjust_flags: a combination of adjustment flags
 *
 * Setup the CRTC modesetting timing parameters for @p, adjusting if necessary.
 *
 * - The CRTC_INTERLACE_HALVE_V flag can be used to halve vertical timings of
 *   interlaced modes.
 * - The CRTC_STEREO_DOUBLE flag can be used to compute the timings for
 *   buffers containing two eyes (only adjust the timings when needed, eg. for
 *   "frame packing" or "side by side full").
 * - The CRTC_NO_DBLSCAN and CRTC_NO_VSCAN flags request that adjustment *not*
 *   be performed for doublescan and vscan > 1 modes respectively.
 */
void drm_mode_set_crtcinfo(struct drm_display_mode *p, int adjust_flags)
{
        if (!p)
                return;

        p->crtc_clock = p->clock;
        p->crtc_hdisplay = p->hdisplay;
        p->crtc_hsync_start = p->hsync_start;
        p->crtc_hsync_end = p->hsync_end;
        p->crtc_htotal = p->htotal;
        p->crtc_hskew = p->hskew;
        p->crtc_vdisplay = p->vdisplay;
        p->crtc_vsync_start = p->vsync_start;
        p->crtc_vsync_end = p->vsync_end;
        p->crtc_vtotal = p->vtotal;

        if (p->flags & DRM_MODE_FLAG_INTERLACE) {
                if (adjust_flags & CRTC_INTERLACE_HALVE_V) {
                        p->crtc_vdisplay /= 2;
                        p->crtc_vsync_start /= 2;
                        p->crtc_vsync_end /= 2;
                        p->crtc_vtotal /= 2;
                }
        }

        if (!(adjust_flags & CRTC_NO_DBLSCAN)) {
                if (p->flags & DRM_MODE_FLAG_DBLSCAN) {
                        p->crtc_vdisplay *= 2;
                        p->crtc_vsync_start *= 2;
                        p->crtc_vsync_end *= 2;
                        p->crtc_vtotal *= 2;
                }
        }

        if (!(adjust_flags & CRTC_NO_VSCAN)) {
                if (p->vscan > 1) {
                        p->crtc_vdisplay *= p->vscan;
                        p->crtc_vsync_start *= p->vscan;
                        p->crtc_vsync_end *= p->vscan;
                        p->crtc_vtotal *= p->vscan;
                }
        }

        if (adjust_flags & CRTC_STEREO_DOUBLE) {
                unsigned int layout = p->flags & DRM_MODE_FLAG_3D_MASK;

                switch (layout) {
                case DRM_MODE_FLAG_3D_FRAME_PACKING:
                        p->crtc_clock *= 2;
                        p->crtc_vdisplay += p->crtc_vtotal;
                        p->crtc_vsync_start += p->crtc_vtotal;
                        p->crtc_vsync_end += p->crtc_vtotal;
                        p->crtc_vtotal += p->crtc_vtotal;
                        break;
                }
        }

        p->crtc_vblank_start = min(p->crtc_vsync_start, p->crtc_vdisplay);
        p->crtc_vblank_end = max(p->crtc_vsync_end, p->crtc_vtotal);
        p->crtc_hblank_start = min(p->crtc_hsync_start, p->crtc_hdisplay);
        p->crtc_hblank_end = max(p->crtc_hsync_end, p->crtc_htotal);
}
EXPORT_SYMBOL(drm_mode_set_crtcinfo);

/**
 * drm_mode_copy - copy the mode
 * @dst: mode to overwrite
 * @src: mode to copy
 *
 * Copy an existing mode into another mode, preserving the
 * list head of the destination mode.
 */
void drm_mode_copy(struct drm_display_mode *dst, const struct drm_display_mode *src)
{
        struct list_head head = dst->head;

        *dst = *src;
        dst->head = head;
}
EXPORT_SYMBOL(drm_mode_copy);

/**
 * drm_mode_init - initialize the mode from another mode
 * @dst: mode to overwrite
 * @src: mode to copy
 *
 * Copy an existing mode into another mode, zeroing the
 * list head of the destination mode. Typically used
 * to guarantee the list head is not left with stack
 * garbage in on-stack modes.
 */
void drm_mode_init(struct drm_display_mode *dst, const struct drm_display_mode *src)
{
        memset(dst, 0, sizeof(*dst));
        drm_mode_copy(dst, src);
}
EXPORT_SYMBOL(drm_mode_init);

/**
 * drm_mode_duplicate - allocate and duplicate an existing mode
 * @dev: drm_device to allocate the duplicated mode for
 * @mode: mode to duplicate
 *
 * Just allocate a new mode, copy the existing mode into it, and return
 * a pointer to it.  Used to create new instances of established modes.
 *
 * Returns:
 * Pointer to duplicated mode on success, NULL on error.
 */
struct drm_display_mode *drm_mode_duplicate(struct drm_device *dev,
                                            const struct drm_display_mode *mode)
{
        struct drm_display_mode *nmode;

        nmode = drm_mode_create(dev);
        if (!nmode)
                return NULL;

        drm_mode_copy(nmode, mode);

        return nmode;
}
EXPORT_SYMBOL(drm_mode_duplicate);

static bool drm_mode_match_timings(const struct drm_display_mode *mode1,
                                   const struct drm_display_mode *mode2)
{
        return mode1->hdisplay == mode2->hdisplay &&
                mode1->hsync_start == mode2->hsync_start &&
                mode1->hsync_end == mode2->hsync_end &&
                mode1->htotal == mode2->htotal &&
                mode1->hskew == mode2->hskew &&
                mode1->vdisplay == mode2->vdisplay &&
                mode1->vsync_start == mode2->vsync_start &&
                mode1->vsync_end == mode2->vsync_end &&
                mode1->vtotal == mode2->vtotal &&
                mode1->vscan == mode2->vscan;
}

static bool drm_mode_match_clock(const struct drm_display_mode *mode1,
                                  const struct drm_display_mode *mode2)
{
        /*
         * do clock check convert to PICOS
         * so fb modes get matched the same
         */
        if (mode1->clock && mode2->clock)
                return KHZ2PICOS(mode1->clock) == KHZ2PICOS(mode2->clock);
        else
                return mode1->clock == mode2->clock;
}

static bool drm_mode_match_flags(const struct drm_display_mode *mode1,
                                 const struct drm_display_mode *mode2)
{
        return (mode1->flags & ~DRM_MODE_FLAG_3D_MASK) ==
                (mode2->flags & ~DRM_MODE_FLAG_3D_MASK);
}

static bool drm_mode_match_3d_flags(const struct drm_display_mode *mode1,
                                    const struct drm_display_mode *mode2)
{
        return (mode1->flags & DRM_MODE_FLAG_3D_MASK) ==
                (mode2->flags & DRM_MODE_FLAG_3D_MASK);
}

static bool drm_mode_match_aspect_ratio(const struct drm_display_mode *mode1,
                                        const struct drm_display_mode *mode2)
{
        return mode1->picture_aspect_ratio == mode2->picture_aspect_ratio;
}

/**
 * drm_mode_match - test modes for (partial) equality
 * @mode1: first mode
 * @mode2: second mode
 * @match_flags: which parts need to match (DRM_MODE_MATCH_*)
 *
 * Check to see if @mode1 and @mode2 are equivalent.
 *
 * Returns:
 * True if the modes are (partially) equal, false otherwise.
 */
bool drm_mode_match(const struct drm_display_mode *mode1,
                    const struct drm_display_mode *mode2,
                    unsigned int match_flags)
{
        if (!mode1 && !mode2)
                return true;

        if (!mode1 || !mode2)
                return false;

        if (match_flags & DRM_MODE_MATCH_TIMINGS &&
            !drm_mode_match_timings(mode1, mode2))
                return false;

        if (match_flags & DRM_MODE_MATCH_CLOCK &&
            !drm_mode_match_clock(mode1, mode2))
                return false;

        if (match_flags & DRM_MODE_MATCH_FLAGS &&
            !drm_mode_match_flags(mode1, mode2))
                return false;

        if (match_flags & DRM_MODE_MATCH_3D_FLAGS &&
            !drm_mode_match_3d_flags(mode1, mode2))
                return false;

        if (match_flags & DRM_MODE_MATCH_ASPECT_RATIO &&
            !drm_mode_match_aspect_ratio(mode1, mode2))
                return false;

        return true;
}
EXPORT_SYMBOL(drm_mode_match);

/**
 * drm_mode_equal - test modes for equality
 * @mode1: first mode
 * @mode2: second mode
 *
 * Check to see if @mode1 and @mode2 are equivalent.
 *
 * Returns:
 * True if the modes are equal, false otherwise.
 */
bool drm_mode_equal(const struct drm_display_mode *mode1,
                    const struct drm_display_mode *mode2)
{
        return drm_mode_match(mode1, mode2,
                              DRM_MODE_MATCH_TIMINGS |
                              DRM_MODE_MATCH_CLOCK |
                              DRM_MODE_MATCH_FLAGS |
                              DRM_MODE_MATCH_3D_FLAGS|
                              DRM_MODE_MATCH_ASPECT_RATIO);
}
EXPORT_SYMBOL(drm_mode_equal);

/**
 * drm_mode_equal_no_clocks - test modes for equality
 * @mode1: first mode
 * @mode2: second mode
 *
 * Check to see if @mode1 and @mode2 are equivalent, but
 * don't check the pixel clocks.
 *
 * Returns:
 * True if the modes are equal, false otherwise.
 */
bool drm_mode_equal_no_clocks(const struct drm_display_mode *mode1,
                              const struct drm_display_mode *mode2)
{
        return drm_mode_match(mode1, mode2,
                              DRM_MODE_MATCH_TIMINGS |
                              DRM_MODE_MATCH_FLAGS |
                              DRM_MODE_MATCH_3D_FLAGS);
}
EXPORT_SYMBOL(drm_mode_equal_no_clocks);

/**
 * drm_mode_equal_no_clocks_no_stereo - test modes for equality
 * @mode1: first mode
 * @mode2: second mode
 *
 * Check to see if @mode1 and @mode2 are equivalent, but
 * don't check the pixel clocks nor the stereo layout.
 *
 * Returns:
 * True if the modes are equal, false otherwise.
 */
bool drm_mode_equal_no_clocks_no_stereo(const struct drm_display_mode *mode1,
                                        const struct drm_display_mode *mode2)
{
        return drm_mode_match(mode1, mode2,
                              DRM_MODE_MATCH_TIMINGS |
                              DRM_MODE_MATCH_FLAGS);
}
EXPORT_SYMBOL(drm_mode_equal_no_clocks_no_stereo);

static enum drm_mode_status
drm_mode_validate_basic(const struct drm_display_mode *mode)
{
        if (mode->type & ~DRM_MODE_TYPE_ALL)
                return MODE_BAD;

        if (mode->flags & ~DRM_MODE_FLAG_ALL)
                return MODE_BAD;

        if ((mode->flags & DRM_MODE_FLAG_3D_MASK) > DRM_MODE_FLAG_3D_MAX)
                return MODE_BAD;

        if (mode->clock == 0)
                return MODE_CLOCK_LOW;

        if (mode->hdisplay == 0 ||
            mode->hsync_start < mode->hdisplay ||
            mode->hsync_end < mode->hsync_start ||
            mode->htotal < mode->hsync_end)
                return MODE_H_ILLEGAL;

        if (mode->vdisplay == 0 ||
            mode->vsync_start < mode->vdisplay ||
            mode->vsync_end < mode->vsync_start ||
            mode->vtotal < mode->vsync_end)
                return MODE_V_ILLEGAL;

        return MODE_OK;
}

/**
 * drm_mode_validate_driver - make sure the mode is somewhat sane
 * @dev: drm device
 * @mode: mode to check
 *
 * First do basic validation on the mode, and then allow the driver
 * to check for device/driver specific limitations via the optional
 * &drm_mode_config_helper_funcs.mode_valid hook.
 *
 * Returns:
 * The mode status
 */
enum drm_mode_status
drm_mode_validate_driver(struct drm_device *dev,
                        const struct drm_display_mode *mode)
{
        enum drm_mode_status status;

        status = drm_mode_validate_basic(mode);
        if (status != MODE_OK)
                return status;

        if (dev->mode_config.funcs->mode_valid)
                return dev->mode_config.funcs->mode_valid(dev, mode);
        else
                return MODE_OK;
}
EXPORT_SYMBOL(drm_mode_validate_driver);

/**
 * drm_mode_validate_size - make sure modes adhere to size constraints
 * @mode: mode to check
 * @maxX: maximum width
 * @maxY: maximum height
 *
 * This function is a helper which can be used to validate modes against size
 * limitations of the DRM device/connector. If a mode is too big its status
 * member is updated with the appropriate validation failure code. The list
 * itself is not changed.
 *
 * Returns:
 * The mode status
 */
enum drm_mode_status
drm_mode_validate_size(const struct drm_display_mode *mode,
                       int maxX, int maxY)
{
        if (maxX > 0 && mode->hdisplay > maxX)
                return MODE_VIRTUAL_X;

        if (maxY > 0 && mode->vdisplay > maxY)
                return MODE_VIRTUAL_Y;

        return MODE_OK;
}
EXPORT_SYMBOL(drm_mode_validate_size);

/**
 * drm_mode_validate_ycbcr420 - add 'ycbcr420-only' modes only when allowed
 * @mode: mode to check
 * @connector: drm connector under action
 *
 * This function is a helper which can be used to filter out any YCBCR420
 * only mode, when the source doesn't support it.
 *
 * Returns:
 * The mode status
 */
enum drm_mode_status
drm_mode_validate_ycbcr420(const struct drm_display_mode *mode,
                           struct drm_connector *connector)
{
        if (!connector->ycbcr_420_allowed &&
            drm_mode_is_420_only(&connector->display_info, mode))
                return MODE_NO_420;

        return MODE_OK;
}
EXPORT_SYMBOL(drm_mode_validate_ycbcr420);

#define MODE_STATUS(status) [MODE_ ## status + 3] = #status

static const char * const drm_mode_status_names[] = {
        MODE_STATUS(OK),
        MODE_STATUS(HSYNC),
        MODE_STATUS(VSYNC),
        MODE_STATUS(H_ILLEGAL),
        MODE_STATUS(V_ILLEGAL),
        MODE_STATUS(BAD_WIDTH),
        MODE_STATUS(NOMODE),
        MODE_STATUS(NO_INTERLACE),
        MODE_STATUS(NO_DBLESCAN),
        MODE_STATUS(NO_VSCAN),
        MODE_STATUS(MEM),
        MODE_STATUS(VIRTUAL_X),
        MODE_STATUS(VIRTUAL_Y),
        MODE_STATUS(MEM_VIRT),
        MODE_STATUS(NOCLOCK),
        MODE_STATUS(CLOCK_HIGH),
        MODE_STATUS(CLOCK_LOW),
        MODE_STATUS(CLOCK_RANGE),
        MODE_STATUS(BAD_HVALUE),
        MODE_STATUS(BAD_VVALUE),
        MODE_STATUS(BAD_VSCAN),
        MODE_STATUS(HSYNC_NARROW),
        MODE_STATUS(HSYNC_WIDE),
        MODE_STATUS(HBLANK_NARROW),
        MODE_STATUS(HBLANK_WIDE),
        MODE_STATUS(VSYNC_NARROW),
        MODE_STATUS(VSYNC_WIDE),
        MODE_STATUS(VBLANK_NARROW),
        MODE_STATUS(VBLANK_WIDE),
        MODE_STATUS(PANEL),
        MODE_STATUS(INTERLACE_WIDTH),
        MODE_STATUS(ONE_WIDTH),
        MODE_STATUS(ONE_HEIGHT),
        MODE_STATUS(ONE_SIZE),
        MODE_STATUS(NO_REDUCED),
        MODE_STATUS(NO_STEREO),
        MODE_STATUS(NO_420),
        MODE_STATUS(STALE),
        MODE_STATUS(BAD),
        MODE_STATUS(ERROR),
};

#undef MODE_STATUS

const char *drm_get_mode_status_name(enum drm_mode_status status)
{
        int index = status + 3;

        if (WARN_ON(index < 0 || index >= ARRAY_SIZE(drm_mode_status_names)))
                return "";

        return drm_mode_status_names[index];
}

/**
 * drm_mode_prune_invalid - remove invalid modes from mode list
 * @dev: DRM device
 * @mode_list: list of modes to check
 * @verbose: be verbose about it
 *
 * This helper function can be used to prune a display mode list after
 * validation has been completed. All modes whose status is not MODE_OK will be
 * removed from the list, and if @verbose the status code and mode name is also
 * printed to dmesg.
 */
void drm_mode_prune_invalid(struct drm_device *dev,
                            struct list_head *mode_list, bool verbose)
{
        struct drm_display_mode *mode, *t;

        list_for_each_entry_safe(mode, t, mode_list, head) {
                if (mode->status != MODE_OK) {
                        list_del(&mode->head);
                        if (mode->type & DRM_MODE_TYPE_USERDEF) {
                                drm_warn(dev, "User-defined mode not supported: "
                                         DRM_MODE_FMT "\n", DRM_MODE_ARG(mode));
                        }
                        if (verbose) {
                                drm_dbg_kms(dev, "Rejected mode: " DRM_MODE_FMT " (%s)\n",
                                            DRM_MODE_ARG(mode), drm_get_mode_status_name(mode->status));
                        }
                        drm_mode_destroy(dev, mode);
                }
        }
}
EXPORT_SYMBOL(drm_mode_prune_invalid);

/**
 * drm_mode_compare - compare modes for favorability
 * @priv: unused
 * @lh_a: list_head for first mode
 * @lh_b: list_head for second mode
 *
 * Compare two modes, given by @lh_a and @lh_b, returning a value indicating
 * which is better.
 *
 * Returns:
 * Negative if @lh_a is better than @lh_b, zero if they're equivalent, or
 * positive if @lh_b is better than @lh_a.
 */
static int drm_mode_compare(void *priv, const struct list_head *lh_a,
                            const struct list_head *lh_b)
{
        struct drm_display_mode *a = list_entry(lh_a, struct drm_display_mode, head);
        struct drm_display_mode *b = list_entry(lh_b, struct drm_display_mode, head);
        int diff;

        diff = ((b->type & DRM_MODE_TYPE_PREFERRED) != 0) -
                ((a->type & DRM_MODE_TYPE_PREFERRED) != 0);
        if (diff)
                return diff;
        diff = b->hdisplay * b->vdisplay - a->hdisplay * a->vdisplay;
        if (diff)
                return diff;

        diff = drm_mode_vrefresh(b) - drm_mode_vrefresh(a);
        if (diff)
                return diff;

        diff = b->clock - a->clock;
        return diff;
}

/**
 * drm_mode_sort - sort mode list
 * @mode_list: list of drm_display_mode structures to sort
 *
 * Sort @mode_list by favorability, moving good modes to the head of the list.
 */
void drm_mode_sort(struct list_head *mode_list)
{
        list_sort(NULL, mode_list, drm_mode_compare);
}
EXPORT_SYMBOL(drm_mode_sort);

/**
 * drm_connector_list_update - update the mode list for the connector
 * @connector: the connector to update
 *
 * This moves the modes from the @connector probed_modes list
 * to the actual mode list. It compares the probed mode against the current
 * list and only adds different/new modes.
 *
 * This is just a helper functions doesn't validate any modes itself and also
 * doesn't prune any invalid modes. Callers need to do that themselves.
 */
void drm_connector_list_update(struct drm_connector *connector)
{
        struct drm_display_mode *pmode, *pt;

        WARN_ON(!mutex_is_locked(&connector->dev->mode_config.mutex));

        list_for_each_entry_safe(pmode, pt, &connector->probed_modes, head) {
                struct drm_display_mode *mode;
                bool found_it = false;

                /* go through current modes checking for the new probed mode */
                list_for_each_entry(mode, &connector->modes, head) {
                        if (!drm_mode_equal(pmode, mode))
                                continue;

                        found_it = true;

                        /*
                         * If the old matching mode is stale (ie. left over
                         * from a previous probe) just replace it outright.
                         * Otherwise just merge the type bits between all
                         * equal probed modes.
                         *
                         * If two probed modes are considered equal, pick the
                         * actual timings from the one that's marked as
                         * preferred (in case the match isn't 100%). If
                         * multiple or zero preferred modes are present, favor
                         * the mode added to the probed_modes list first.
                         */
                        if (mode->status == MODE_STALE) {
                                drm_mode_copy(mode, pmode);
                        } else if ((mode->type & DRM_MODE_TYPE_PREFERRED) == 0 &&
                                   (pmode->type & DRM_MODE_TYPE_PREFERRED) != 0) {
                                pmode->type |= mode->type;
                                drm_mode_copy(mode, pmode);
                        } else {
                                mode->type |= pmode->type;
                        }

                        list_del(&pmode->head);
                        drm_mode_destroy(connector->dev, pmode);
                        break;
                }

                if (!found_it) {
                        list_move_tail(&pmode->head, &connector->modes);
                }
        }
}
EXPORT_SYMBOL(drm_connector_list_update);

static int drm_mode_parse_cmdline_bpp(const char *str, char **end_ptr,
                                      struct drm_cmdline_mode *mode)
{
        unsigned int bpp;

        if (str[0] != '-')
                return -EINVAL;

        str++;
        bpp = simple_strtol(str, end_ptr, 10);
        if (*end_ptr == str)
                return -EINVAL;

        mode->bpp = bpp;
        mode->bpp_specified = true;

        return 0;
}

static int drm_mode_parse_cmdline_refresh(const char *str, char **end_ptr,
                                          struct drm_cmdline_mode *mode)
{
        unsigned int refresh;

        if (str[0] != '@')
                return -EINVAL;

        str++;
        refresh = simple_strtol(str, end_ptr, 10);
        if (*end_ptr == str)
                return -EINVAL;

        mode->refresh = refresh;
        mode->refresh_specified = true;

        return 0;
}

static int drm_mode_parse_cmdline_extra(const char *str, int length,
                                        bool freestanding,
                                        const struct drm_connector *connector,
                                        struct drm_cmdline_mode *mode)
{
        int i;

        for (i = 0; i < length; i++) {
                switch (str[i]) {
                case 'i':
                        if (freestanding)
                                return -EINVAL;

                        mode->interlace = true;
                        break;
                case 'm':
                        if (freestanding)
                                return -EINVAL;

                        mode->margins = true;
                        break;
                case 'D':
                        if (mode->force != DRM_FORCE_UNSPECIFIED)
                                return -EINVAL;

                        if ((connector->connector_type != DRM_MODE_CONNECTOR_DVII) &&
                            (connector->connector_type != DRM_MODE_CONNECTOR_HDMIB))
                                mode->force = DRM_FORCE_ON;
                        else
                                mode->force = DRM_FORCE_ON_DIGITAL;
                        break;
                case 'd':
                        if (mode->force != DRM_FORCE_UNSPECIFIED)
                                return -EINVAL;

                        mode->force = DRM_FORCE_OFF;
                        break;
                case 'e':
                        if (mode->force != DRM_FORCE_UNSPECIFIED)
                                return -EINVAL;

                        mode->force = DRM_FORCE_ON;
                        break;
                default:
                        return -EINVAL;
                }
        }

        return 0;
}

static int drm_mode_parse_cmdline_res_mode(const char *str, unsigned int length,
                                           bool extras,
                                           const struct drm_connector *connector,
                                           struct drm_cmdline_mode *mode)
{
        const char *str_start = str;
        bool rb = false, cvt = false;
        int xres = 0, yres = 0;
        int remaining, i;
        char *end_ptr;

        xres = simple_strtol(str, &end_ptr, 10);
        if (end_ptr == str)
                return -EINVAL;

        if (end_ptr[0] != 'x')
                return -EINVAL;
        end_ptr++;

        str = end_ptr;
        yres = simple_strtol(str, &end_ptr, 10);
        if (end_ptr == str)
                return -EINVAL;

        remaining = length - (end_ptr - str_start);
        if (remaining < 0)
                return -EINVAL;

        for (i = 0; i < remaining; i++) {
                switch (end_ptr[i]) {
                case 'M':
                        cvt = true;
                        break;
                case 'R':
                        rb = true;
                        break;
                default:
                        /*
                         * Try to pass that to our extras parsing
                         * function to handle the case where the
                         * extras are directly after the resolution
                         */
                        if (extras) {
                                int ret = drm_mode_parse_cmdline_extra(end_ptr + i,
                                                                       1,
                                                                       false,
                                                                       connector,
                                                                       mode);
                                if (ret)
                                        return ret;
                        } else {
                                return -EINVAL;
                        }
                }
        }

        mode->xres = xres;
        mode->yres = yres;
        mode->cvt = cvt;
        mode->rb = rb;

        return 0;
}

static int drm_mode_parse_cmdline_int(const char *delim, unsigned int *int_ret)
{
        const char *value;
        char *endp;

        /*
         * delim must point to the '=', otherwise it is a syntax error and
         * if delim points to the terminating zero, then delim + 1 will point
         * past the end of the string.
         */
        if (*delim != '=')
                return -EINVAL;

        value = delim + 1;
        *int_ret = simple_strtol(value, &endp, 10);

        /* Make sure we have parsed something */
        if (endp == value)
                return -EINVAL;

        return 0;
}

static int drm_mode_parse_panel_orientation(const char *delim,
                                            struct drm_cmdline_mode *mode)
{
        const char *value;

        if (*delim != '=')
                return -EINVAL;

        value = delim + 1;
        delim = strchr(value, ',');
        if (!delim)
                delim = value + strlen(value);

        if (!strncmp(value, "normal", delim - value))
                mode->panel_orientation = DRM_MODE_PANEL_ORIENTATION_NORMAL;
        else if (!strncmp(value, "upside_down", delim - value))
                mode->panel_orientation = DRM_MODE_PANEL_ORIENTATION_BOTTOM_UP;
        else if (!strncmp(value, "left_side_up", delim - value))
                mode->panel_orientation = DRM_MODE_PANEL_ORIENTATION_LEFT_UP;
        else if (!strncmp(value, "right_side_up", delim - value))
                mode->panel_orientation = DRM_MODE_PANEL_ORIENTATION_RIGHT_UP;
        else
                return -EINVAL;

        return 0;
}

static int drm_mode_parse_tv_mode(const char *delim,
                                  struct drm_cmdline_mode *mode)
{
        const char *value;
        int ret;

        if (*delim != '=')
                return -EINVAL;

        value = delim + 1;
        delim = strchr(value, ',');
        if (!delim)
                delim = value + strlen(value);

        ret = drm_get_tv_mode_from_name(value, delim - value);
        if (ret < 0)
                return ret;

        mode->tv_mode_specified = true;
        mode->tv_mode = ret;

        return 0;
}

static int drm_mode_parse_cmdline_options(const char *str,
                                          bool freestanding,
                                          const struct drm_connector *connector,
                                          struct drm_cmdline_mode *mode)
{
        unsigned int deg, margin, rotation = 0;
        const char *delim, *option, *sep;

        option = str;
        do {
                delim = strchr(option, '=');
                if (!delim) {
                        delim = strchr(option, ',');

                        if (!delim)
                                delim = option + strlen(option);
                }

                if (!strncmp(option, "rotate", delim - option)) {
                        if (drm_mode_parse_cmdline_int(delim, &deg))
                                return -EINVAL;

                        switch (deg) {
                        case 0:
                                rotation |= DRM_MODE_ROTATE_0;
                                break;

                        case 90:
                                rotation |= DRM_MODE_ROTATE_90;
                                break;

                        case 180:
                                rotation |= DRM_MODE_ROTATE_180;
                                break;

                        case 270:
                                rotation |= DRM_MODE_ROTATE_270;
                                break;

                        default:
                                return -EINVAL;
                        }
                } else if (!strncmp(option, "reflect_x", delim - option)) {
                        rotation |= DRM_MODE_REFLECT_X;
                } else if (!strncmp(option, "reflect_y", delim - option)) {
                        rotation |= DRM_MODE_REFLECT_Y;
                } else if (!strncmp(option, "margin_right", delim - option)) {
                        if (drm_mode_parse_cmdline_int(delim, &margin))
                                return -EINVAL;

                        mode->tv_margins.right = margin;
                } else if (!strncmp(option, "margin_left", delim - option)) {
                        if (drm_mode_parse_cmdline_int(delim, &margin))
                                return -EINVAL;

                        mode->tv_margins.left = margin;
                } else if (!strncmp(option, "margin_top", delim - option)) {
                        if (drm_mode_parse_cmdline_int(delim, &margin))
                                return -EINVAL;

                        mode->tv_margins.top = margin;
                } else if (!strncmp(option, "margin_bottom", delim - option)) {
                        if (drm_mode_parse_cmdline_int(delim, &margin))
                                return -EINVAL;

                        mode->tv_margins.bottom = margin;
                } else if (!strncmp(option, "panel_orientation", delim - option)) {
                        if (drm_mode_parse_panel_orientation(delim, mode))
                                return -EINVAL;
                } else if (!strncmp(option, "tv_mode", delim - option)) {
                        if (drm_mode_parse_tv_mode(delim, mode))
                                return -EINVAL;
                } else {
                        return -EINVAL;
                }
                sep = strchr(delim, ',');
                option = sep + 1;
        } while (sep);

        if (rotation && freestanding)
                return -EINVAL;

        if (!(rotation & DRM_MODE_ROTATE_MASK))
                rotation |= DRM_MODE_ROTATE_0;

        /* Make sure there is exactly one rotation defined */
        if (!is_power_of_2(rotation & DRM_MODE_ROTATE_MASK))
                return -EINVAL;

        mode->rotation_reflection = rotation;

        return 0;
}

struct drm_named_mode {
        const char *name;
        unsigned int pixel_clock_khz;
        unsigned int xres;
        unsigned int yres;
        unsigned int flags;
        unsigned int tv_mode;
};

#define NAMED_MODE(_name, _pclk, _x, _y, _flags, _mode)        \
        {                                                \
                .name = _name,                                \
                .pixel_clock_khz = _pclk,                \
                .xres = _x,                                \
                .yres = _y,                                \
                .flags = _flags,                        \
                .tv_mode = _mode,                        \
        }

static const struct drm_named_mode drm_named_modes[] = {
        NAMED_MODE("NTSC", 13500, 720, 480, DRM_MODE_FLAG_INTERLACE, DRM_MODE_TV_MODE_NTSC),
        NAMED_MODE("NTSC-J", 13500, 720, 480, DRM_MODE_FLAG_INTERLACE, DRM_MODE_TV_MODE_NTSC_J),
        NAMED_MODE("PAL", 13500, 720, 576, DRM_MODE_FLAG_INTERLACE, DRM_MODE_TV_MODE_PAL),
        NAMED_MODE("PAL-M", 13500, 720, 480, DRM_MODE_FLAG_INTERLACE, DRM_MODE_TV_MODE_PAL_M),
};

static int drm_mode_parse_cmdline_named_mode(const char *name,
                                             unsigned int name_end,
                                             struct drm_cmdline_mode *cmdline_mode)
{
        unsigned int i;

        if (!name_end)
                return 0;

        /* If the name starts with a digit, it's not a named mode */
        if (isdigit(name[0]))
                return 0;

        /*
         * If there's an equal sign in the name, the command-line
         * contains only an option and no mode.
         */
        if (strnchr(name, name_end, '='))
                return 0;

        /* The connection status extras can be set without a mode. */
        if (name_end == 1 &&
            (name[0] == 'd' || name[0] == 'D' || name[0] == 'e'))
                return 0;

        /*
         * We're sure we're a named mode at this point, iterate over the
         * list of modes we're aware of.
         */
        for (i = 0; i < ARRAY_SIZE(drm_named_modes); i++) {
                const struct drm_named_mode *mode = &drm_named_modes[i];
                int ret;

                ret = str_has_prefix(name, mode->name);
                if (ret != name_end)
                        continue;

                strscpy(cmdline_mode->name, mode->name, sizeof(cmdline_mode->name));
                cmdline_mode->pixel_clock = mode->pixel_clock_khz;
                cmdline_mode->xres = mode->xres;
                cmdline_mode->yres = mode->yres;
                cmdline_mode->interlace = !!(mode->flags & DRM_MODE_FLAG_INTERLACE);
                cmdline_mode->tv_mode = mode->tv_mode;
                cmdline_mode->tv_mode_specified = true;
                cmdline_mode->specified = true;

                return 1;
        }

        return -EINVAL;
}

/**
 * drm_mode_parse_command_line_for_connector - parse command line modeline for connector
 * @mode_option: optional per connector mode option
 * @connector: connector to parse modeline for
 * @mode: preallocated drm_cmdline_mode structure to fill out
 *
 * This parses @mode_option command line modeline for modes and options to
 * configure the connector.
 *
 * This uses the same parameters as the fb modedb.c, except for an extra
 * force-enable, force-enable-digital and force-disable bit at the end::
 *
 *        <xres>x<yres>[M][R][-<bpp>][@<refresh>][i][m][eDd]
 *
 * Additionals options can be provided following the mode, using a comma to
 * separate each option. Valid options can be found in
 * Documentation/fb/modedb.rst.
 *
 * The intermediate drm_cmdline_mode structure is required to store additional
 * options from the command line modline like the force-enable/disable flag.
 *
 * Returns:
 * True if a valid modeline has been parsed, false otherwise.
 */
bool drm_mode_parse_command_line_for_connector(const char *mode_option,
                                               const struct drm_connector *connector,
                                               struct drm_cmdline_mode *mode)
{
        const char *name;
        bool freestanding = false, parse_extras = false;
        unsigned int bpp_off = 0, refresh_off = 0, options_off = 0;
        unsigned int mode_end = 0;
        const char *bpp_ptr = NULL, *refresh_ptr = NULL, *extra_ptr = NULL;
        const char *options_ptr = NULL;
        char *bpp_end_ptr = NULL, *refresh_end_ptr = NULL;
        int len, ret;

        memset(mode, 0, sizeof(*mode));
        mode->panel_orientation = DRM_MODE_PANEL_ORIENTATION_UNKNOWN;

        if (!mode_option)
                return false;

        name = mode_option;

        /* Locate the start of named options */
        options_ptr = strchr(name, ',');
        if (options_ptr)
                options_off = options_ptr - name;
        else
                options_off = strlen(name);

        /* Try to locate the bpp and refresh specifiers, if any */
        bpp_ptr = strnchr(name, options_off, '-');
        while (bpp_ptr && !isdigit(bpp_ptr[1]))
                bpp_ptr = strnchr(bpp_ptr + 1, options_off, '-');
        if (bpp_ptr)
                bpp_off = bpp_ptr - name;

        refresh_ptr = strnchr(name, options_off, '@');
        if (refresh_ptr)
                refresh_off = refresh_ptr - name;

        /* Locate the end of the name / resolution, and parse it */
        if (bpp_ptr) {
                mode_end = bpp_off;
        } else if (refresh_ptr) {
                mode_end = refresh_off;
        } else if (options_ptr) {
                mode_end = options_off;
                parse_extras = true;
        } else {
                mode_end = strlen(name);
                parse_extras = true;
        }

        if (!mode_end)
                return false;

        ret = drm_mode_parse_cmdline_named_mode(name, mode_end, mode);
        if (ret < 0)
                return false;

        /*
         * Having a mode that starts by a letter (and thus is named) and
         * an at-sign (used to specify a refresh rate) is disallowed.
         */
        if (ret && refresh_ptr)
                return false;

        /* No named mode? Check for a normal mode argument, e.g. 1024x768 */
        if (!mode->specified && isdigit(name[0])) {
                ret = drm_mode_parse_cmdline_res_mode(name, mode_end,
                                                      parse_extras,
                                                      connector,
                                                      mode);
                if (ret)
                        return false;

                mode->specified = true;
        }

        /* No mode? Check for freestanding extras and/or options */
        if (!mode->specified) {
                unsigned int len = strlen(mode_option);

                if (bpp_ptr || refresh_ptr)
                        return false; /* syntax error */

                if (len == 1 || (len >= 2 && mode_option[1] == ','))
                        extra_ptr = mode_option;
                else
                        options_ptr = mode_option - 1;

                freestanding = true;
        }

        if (bpp_ptr) {
                ret = drm_mode_parse_cmdline_bpp(bpp_ptr, &bpp_end_ptr, mode);
                if (ret)
                        return false;

                mode->bpp_specified = true;
        }

        if (refresh_ptr) {
                ret = drm_mode_parse_cmdline_refresh(refresh_ptr,
                                                     &refresh_end_ptr, mode);
                if (ret)
                        return false;

                mode->refresh_specified = true;
        }

        /*
         * Locate the end of the bpp / refresh, and parse the extras
         * if relevant
         */
        if (bpp_ptr && refresh_ptr)
                extra_ptr = max(bpp_end_ptr, refresh_end_ptr);
        else if (bpp_ptr)
                extra_ptr = bpp_end_ptr;
        else if (refresh_ptr)
                extra_ptr = refresh_end_ptr;

        if (extra_ptr) {
                if (options_ptr)
                        len = options_ptr - extra_ptr;
                else
                        len = strlen(extra_ptr);

                ret = drm_mode_parse_cmdline_extra(extra_ptr, len, freestanding,
                                                   connector, mode);
                if (ret)
                        return false;
        }

        if (options_ptr) {
                ret = drm_mode_parse_cmdline_options(options_ptr + 1,
                                                     freestanding,
                                                     connector, mode);
                if (ret)
                        return false;
        }

        return true;
}
EXPORT_SYMBOL(drm_mode_parse_command_line_for_connector);

static struct drm_display_mode *drm_named_mode(struct drm_device *dev,
                                               struct drm_cmdline_mode *cmd)
{
        unsigned int i;

        for (i = 0; i < ARRAY_SIZE(drm_named_modes); i++) {
                const struct drm_named_mode *named_mode = &drm_named_modes[i];

                if (strcmp(cmd->name, named_mode->name))
                        continue;

                if (!cmd->tv_mode_specified)
                        continue;

                return drm_analog_tv_mode(dev,
                                          named_mode->tv_mode,
                                          named_mode->pixel_clock_khz * 1000,
                                          named_mode->xres,
                                          named_mode->yres,
                                          named_mode->flags & DRM_MODE_FLAG_INTERLACE);
        }

        return NULL;
}

/**
 * drm_mode_create_from_cmdline_mode - convert a command line modeline into a DRM display mode
 * @dev: DRM device to create the new mode for
 * @cmd: input command line modeline
 *
 * Returns:
 * Pointer to converted mode on success, NULL on error.
 */
struct drm_display_mode *
drm_mode_create_from_cmdline_mode(struct drm_device *dev,
                                  struct drm_cmdline_mode *cmd)
{
        struct drm_display_mode *mode;

        if (cmd->xres == 0 || cmd->yres == 0)
                return NULL;

        if (strlen(cmd->name))
                mode = drm_named_mode(dev, cmd);
        else if (cmd->cvt)
                mode = drm_cvt_mode(dev,
                                    cmd->xres, cmd->yres,
                                    cmd->refresh_specified ? cmd->refresh : 60,
                                    cmd->rb, cmd->interlace,
                                    cmd->margins);
        else
                mode = drm_gtf_mode(dev,
                                    cmd->xres, cmd->yres,
                                    cmd->refresh_specified ? cmd->refresh : 60,
                                    cmd->interlace,
                                    cmd->margins);
        if (!mode)
                return NULL;

        mode->type |= DRM_MODE_TYPE_USERDEF;
        /* fix up 1368x768: GFT/CVT can't express 1366 width due to alignment */
        if (cmd->xres == 1366)
                drm_mode_fixup_1366x768(mode);
        drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
        return mode;
}
EXPORT_SYMBOL(drm_mode_create_from_cmdline_mode);

/**
 * drm_mode_convert_to_umode - convert a drm_display_mode into a modeinfo
 * @out: drm_mode_modeinfo struct to return to the user
 * @in: drm_display_mode to use
 *
 * Convert a drm_display_mode into a drm_mode_modeinfo structure to return to
 * the user.
 */
void drm_mode_convert_to_umode(struct drm_mode_modeinfo *out,
                               const struct drm_display_mode *in)
{
        out->clock = in->clock;
        out->hdisplay = in->hdisplay;
        out->hsync_start = in->hsync_start;
        out->hsync_end = in->hsync_end;
        out->htotal = in->htotal;
        out->hskew = in->hskew;
        out->vdisplay = in->vdisplay;
        out->vsync_start = in->vsync_start;
        out->vsync_end = in->vsync_end;
        out->vtotal = in->vtotal;
        out->vscan = in->vscan;
        out->vrefresh = drm_mode_vrefresh(in);
        out->flags = in->flags;
        out->type = in->type;

        switch (in->picture_aspect_ratio) {
        case HDMI_PICTURE_ASPECT_4_3:
                out->flags |= DRM_MODE_FLAG_PIC_AR_4_3;
                break;
        case HDMI_PICTURE_ASPECT_16_9:
                out->flags |= DRM_MODE_FLAG_PIC_AR_16_9;
                break;
        case HDMI_PICTURE_ASPECT_64_27:
                out->flags |= DRM_MODE_FLAG_PIC_AR_64_27;
                break;
        case HDMI_PICTURE_ASPECT_256_135:
                out->flags |= DRM_MODE_FLAG_PIC_AR_256_135;
                break;
        default:
                WARN(1, "Invalid aspect ratio (0%x) on mode\n",
                     in->picture_aspect_ratio);
                fallthrough;
        case HDMI_PICTURE_ASPECT_NONE:
                out->flags |= DRM_MODE_FLAG_PIC_AR_NONE;
                break;
        }

        strscpy_pad(out->name, in->name, sizeof(out->name));
}

/**
 * drm_mode_convert_umode - convert a modeinfo into a drm_display_mode
 * @dev: drm device
 * @out: drm_display_mode to return to the user
 * @in: drm_mode_modeinfo to use
 *
 * Convert a drm_mode_modeinfo into a drm_display_mode structure to return to
 * the caller.
 *
 * Returns:
 * Zero on success, negative errno on failure.
 */
int drm_mode_convert_umode(struct drm_device *dev,
                           struct drm_display_mode *out,
                           const struct drm_mode_modeinfo *in)
{
        if (in->clock > INT_MAX || in->vrefresh > INT_MAX)
                return -ERANGE;

        out->clock = in->clock;
        out->hdisplay = in->hdisplay;
        out->hsync_start = in->hsync_start;
        out->hsync_end = in->hsync_end;
        out->htotal = in->htotal;
        out->hskew = in->hskew;
        out->vdisplay = in->vdisplay;
        out->vsync_start = in->vsync_start;
        out->vsync_end = in->vsync_end;
        out->vtotal = in->vtotal;
        out->vscan = in->vscan;
        out->flags = in->flags;
        /*
         * Old xf86-video-vmware (possibly others too) used to
         * leave 'type' uninitialized. Just ignore any bits we
         * don't like. It's a just hint after all, and more
         * useful for the kernel->userspace direction anyway.
         */
        out->type = in->type & DRM_MODE_TYPE_ALL;
        strscpy_pad(out->name, in->name, sizeof(out->name));

        /* Clearing picture aspect ratio bits from out flags,
         * as the aspect-ratio information is not stored in
         * flags for kernel-mode, but in picture_aspect_ratio.
         */
        out->flags &= ~DRM_MODE_FLAG_PIC_AR_MASK;

        switch (in->flags & DRM_MODE_FLAG_PIC_AR_MASK) {
        case DRM_MODE_FLAG_PIC_AR_4_3:
                out->picture_aspect_ratio = HDMI_PICTURE_ASPECT_4_3;
                break;
        case DRM_MODE_FLAG_PIC_AR_16_9:
                out->picture_aspect_ratio = HDMI_PICTURE_ASPECT_16_9;
                break;
        case DRM_MODE_FLAG_PIC_AR_64_27:
                out->picture_aspect_ratio = HDMI_PICTURE_ASPECT_64_27;
                break;
        case DRM_MODE_FLAG_PIC_AR_256_135:
                out->picture_aspect_ratio = HDMI_PICTURE_ASPECT_256_135;
                break;
        case DRM_MODE_FLAG_PIC_AR_NONE:
                out->picture_aspect_ratio = HDMI_PICTURE_ASPECT_NONE;
                break;
        default:
                return -EINVAL;
        }

        out->status = drm_mode_validate_driver(dev, out);
        if (out->status != MODE_OK)
                return -EINVAL;

        drm_mode_set_crtcinfo(out, CRTC_INTERLACE_HALVE_V);

        return 0;
}

/**
 * drm_mode_is_420_only - if a given videomode can be only supported in YCBCR420
 * output format
 *
 * @display: display under action
 * @mode: video mode to be tested.
 *
 * Returns:
 * true if the mode can be supported in YCBCR420 format
 * false if not.
 */
bool drm_mode_is_420_only(const struct drm_display_info *display,
                          const struct drm_display_mode *mode)
{
        u8 vic = drm_match_cea_mode(mode);

        return test_bit(vic, display->hdmi.y420_vdb_modes);
}
EXPORT_SYMBOL(drm_mode_is_420_only);

/**
 * drm_mode_is_420_also - if a given videomode can be supported in YCBCR420
 * output format also (along with RGB/YCBCR444/422)
 *
 * @display: display under action.
 * @mode: video mode to be tested.
 *
 * Returns:
 * true if the mode can be support YCBCR420 format
 * false if not.
 */
bool drm_mode_is_420_also(const struct drm_display_info *display,
                          const struct drm_display_mode *mode)
{
        u8 vic = drm_match_cea_mode(mode);

        return test_bit(vic, display->hdmi.y420_cmdb_modes);
}
EXPORT_SYMBOL(drm_mode_is_420_also);
/**
 * drm_mode_is_420 - if a given videomode can be supported in YCBCR420
 * output format
 *
 * @display: display under action.
 * @mode: video mode to be tested.
 *
 * Returns:
 * true if the mode can be supported in YCBCR420 format
 * false if not.
 */
bool drm_mode_is_420(const struct drm_display_info *display,
                     const struct drm_display_mode *mode)
{
        return drm_mode_is_420_only(display, mode) ||
                drm_mode_is_420_also(display, mode);
}
EXPORT_SYMBOL(drm_mode_is_420);

/**
 * drm_set_preferred_mode - Sets the preferred mode of a connector
 * @connector: connector whose mode list should be processed
 * @hpref: horizontal resolution of preferred mode
 * @vpref: vertical resolution of preferred mode
 *
 * Marks a mode as preferred if it matches the resolution specified by @hpref
 * and @vpref.
 */
void drm_set_preferred_mode(struct drm_connector *connector,
                            int hpref, int vpref)
{
        struct drm_display_mode *mode;

        list_for_each_entry(mode, &connector->probed_modes, head) {
                if (mode->hdisplay == hpref &&
                    mode->vdisplay == vpref)
                        mode->type |= DRM_MODE_TYPE_PREFERRED;
        }
}
EXPORT_SYMBOL(drm_set_preferred_mode);

































































































































































































   14 




















   14 





   14 















   11 










    3 



    8 



    8 

















































































   14 








    1 




    2 



   11 




















































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/sch_red.c        Random Early Detection queue.
 *
 * Authors:        Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
 *
 * Changes:
 * J Hadi Salim 980914:        computation fixes
 * Alexey Makarenko <makar@phoenix.kharkov.ua> 990814: qave on idle link was calculated incorrectly.
 * J Hadi Salim 980816:  ECN support
 */

#include <linux/module.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/skbuff.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/inet_ecn.h>
#include <net/red.h>


/*        Parameters, settable by user:
        -----------------------------

        limit                - bytes (must be > qth_max + burst)

        Hard limit on queue length, should be chosen >qth_max
        to allow packet bursts. This parameter does not
        affect the algorithms behaviour and can be chosen
        arbitrarily high (well, less than ram size)
        Really, this limit will never be reached
        if RED works correctly.
 */

struct red_sched_data {
        u32                        limit;                /* HARD maximal queue length */

        unsigned char                flags;
        /* Non-flags in tc_red_qopt.flags. */
        unsigned char                userbits;

        struct timer_list        adapt_timer;
        struct Qdisc                *sch;
        struct red_parms        parms;
        struct red_vars                vars;
        struct red_stats        stats;
        struct Qdisc                *qdisc;
        struct tcf_qevent        qe_early_drop;
        struct tcf_qevent        qe_mark;
};

#define TC_RED_SUPPORTED_FLAGS (TC_RED_HISTORIC_FLAGS | TC_RED_NODROP)

static inline int red_use_ecn(struct red_sched_data *q)
{
        return q->flags & TC_RED_ECN;
}

static inline int red_use_harddrop(struct red_sched_data *q)
{
        return q->flags & TC_RED_HARDDROP;
}

static int red_use_nodrop(struct red_sched_data *q)
{
        return q->flags & TC_RED_NODROP;
}

static int red_enqueue(struct sk_buff *skb, struct Qdisc *sch,
                       struct sk_buff **to_free)
{
        struct red_sched_data *q = qdisc_priv(sch);
        struct Qdisc *child = q->qdisc;
        unsigned int len;
        int ret;

        q->vars.qavg = red_calc_qavg(&q->parms,
                                     &q->vars,
                                     child->qstats.backlog);

        if (red_is_idling(&q->vars))
                red_end_of_idle_period(&q->vars);

        switch (red_action(&q->parms, &q->vars, q->vars.qavg)) {
        case RED_DONT_MARK:
                break;

        case RED_PROB_MARK:
                qdisc_qstats_overlimit(sch);
                if (!red_use_ecn(q)) {
                        q->stats.prob_drop++;
                        goto congestion_drop;
                }

                if (INET_ECN_set_ce(skb)) {
                        q->stats.prob_mark++;
                        skb = tcf_qevent_handle(&q->qe_mark, sch, skb, to_free, &ret);
                        if (!skb)
                                return NET_XMIT_CN | ret;
                } else if (!red_use_nodrop(q)) {
                        q->stats.prob_drop++;
                        goto congestion_drop;
                }

                /* Non-ECT packet in ECN nodrop mode: queue it. */
                break;

        case RED_HARD_MARK:
                qdisc_qstats_overlimit(sch);
                if (red_use_harddrop(q) || !red_use_ecn(q)) {
                        q->stats.forced_drop++;
                        goto congestion_drop;
                }

                if (INET_ECN_set_ce(skb)) {
                        q->stats.forced_mark++;
                        skb = tcf_qevent_handle(&q->qe_mark, sch, skb, to_free, &ret);
                        if (!skb)
                                return NET_XMIT_CN | ret;
                } else if (!red_use_nodrop(q)) {
                        q->stats.forced_drop++;
                        goto congestion_drop;
                }

                /* Non-ECT packet in ECN nodrop mode: queue it. */
                break;
        }

        len = qdisc_pkt_len(skb);
        ret = qdisc_enqueue(skb, child, to_free);
        if (likely(ret == NET_XMIT_SUCCESS)) {
                sch->qstats.backlog += len;
                sch->q.qlen++;
        } else if (net_xmit_drop_count(ret)) {
                q->stats.pdrop++;
                qdisc_qstats_drop(sch);
        }
        return ret;

congestion_drop:
        skb = tcf_qevent_handle(&q->qe_early_drop, sch, skb, to_free, &ret);
        if (!skb)
                return NET_XMIT_CN | ret;

        qdisc_drop(skb, sch, to_free);
        return NET_XMIT_CN;
}

static struct sk_buff *red_dequeue(struct Qdisc *sch)
{
        struct sk_buff *skb;
        struct red_sched_data *q = qdisc_priv(sch);
        struct Qdisc *child = q->qdisc;

        skb = child->dequeue(child);
        if (skb) {
                qdisc_bstats_update(sch, skb);
                qdisc_qstats_backlog_dec(sch, skb);
                sch->q.qlen--;
        } else {
                if (!red_is_idling(&q->vars))
                        red_start_of_idle_period(&q->vars);
        }
        return skb;
}

static struct sk_buff *red_peek(struct Qdisc *sch)
{
        struct red_sched_data *q = qdisc_priv(sch);
        struct Qdisc *child = q->qdisc;

        return child->ops->peek(child);
}

static void red_reset(struct Qdisc *sch)
{
        struct red_sched_data *q = qdisc_priv(sch);

        qdisc_reset(q->qdisc);
        red_restart(&q->vars);
}

static int red_offload(struct Qdisc *sch, bool enable)
{
        struct red_sched_data *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        struct tc_red_qopt_offload opt = {
                .handle = sch->handle,
                .parent = sch->parent,
        };

        if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
                return -EOPNOTSUPP;

        if (enable) {
                opt.command = TC_RED_REPLACE;
                opt.set.min = q->parms.qth_min >> q->parms.Wlog;
                opt.set.max = q->parms.qth_max >> q->parms.Wlog;
                opt.set.probability = q->parms.max_P;
                opt.set.limit = q->limit;
                opt.set.is_ecn = red_use_ecn(q);
                opt.set.is_harddrop = red_use_harddrop(q);
                opt.set.is_nodrop = red_use_nodrop(q);
                opt.set.qstats = &sch->qstats;
        } else {
                opt.command = TC_RED_DESTROY;
        }

        return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_RED, &opt);
}

static void red_destroy(struct Qdisc *sch)
{
        struct red_sched_data *q = qdisc_priv(sch);

        tcf_qevent_destroy(&q->qe_mark, sch);
        tcf_qevent_destroy(&q->qe_early_drop, sch);
        del_timer_sync(&q->adapt_timer);
        red_offload(sch, false);
        qdisc_put(q->qdisc);
}

static const struct nla_policy red_policy[TCA_RED_MAX + 1] = {
        [TCA_RED_UNSPEC] = { .strict_start_type = TCA_RED_FLAGS },
        [TCA_RED_PARMS]        = { .len = sizeof(struct tc_red_qopt) },
        [TCA_RED_STAB]        = { .len = RED_STAB_SIZE },
        [TCA_RED_MAX_P] = { .type = NLA_U32 },
        [TCA_RED_FLAGS] = NLA_POLICY_BITFIELD32(TC_RED_SUPPORTED_FLAGS),
        [TCA_RED_EARLY_DROP_BLOCK] = { .type = NLA_U32 },
        [TCA_RED_MARK_BLOCK] = { .type = NLA_U32 },
};

static int __red_change(struct Qdisc *sch, struct nlattr **tb,
                        struct netlink_ext_ack *extack)
{
        struct Qdisc *old_child = NULL, *child = NULL;
        struct red_sched_data *q = qdisc_priv(sch);
        struct nla_bitfield32 flags_bf;
        struct tc_red_qopt *ctl;
        unsigned char userbits;
        unsigned char flags;
        int err;
        u32 max_P;
        u8 *stab;

        if (tb[TCA_RED_PARMS] == NULL ||
            tb[TCA_RED_STAB] == NULL)
                return -EINVAL;

        max_P = tb[TCA_RED_MAX_P] ? nla_get_u32(tb[TCA_RED_MAX_P]) : 0;

        ctl = nla_data(tb[TCA_RED_PARMS]);
        stab = nla_data(tb[TCA_RED_STAB]);
        if (!red_check_params(ctl->qth_min, ctl->qth_max, ctl->Wlog,
                              ctl->Scell_log, stab))
                return -EINVAL;

        err = red_get_flags(ctl->flags, TC_RED_HISTORIC_FLAGS,
                            tb[TCA_RED_FLAGS], TC_RED_SUPPORTED_FLAGS,
                            &flags_bf, &userbits, extack);
        if (err)
                return err;

        if (ctl->limit > 0) {
                child = fifo_create_dflt(sch, &bfifo_qdisc_ops, ctl->limit,
                                         extack);
                if (IS_ERR(child))
                        return PTR_ERR(child);

                /* child is fifo, no need to check for noop_qdisc */
                qdisc_hash_add(child, true);
        }

        sch_tree_lock(sch);

        flags = (q->flags & ~flags_bf.selector) | flags_bf.value;
        err = red_validate_flags(flags, extack);
        if (err)
                goto unlock_out;

        q->flags = flags;
        q->userbits = userbits;
        q->limit = ctl->limit;
        if (child) {
                qdisc_tree_flush_backlog(q->qdisc);
                old_child = q->qdisc;
                q->qdisc = child;
        }

        red_set_parms(&q->parms,
                      ctl->qth_min, ctl->qth_max, ctl->Wlog,
                      ctl->Plog, ctl->Scell_log,
                      stab,
                      max_P);
        red_set_vars(&q->vars);

        del_timer(&q->adapt_timer);
        if (ctl->flags & TC_RED_ADAPTATIVE)
                mod_timer(&q->adapt_timer, jiffies + HZ/2);

        if (!q->qdisc->q.qlen)
                red_start_of_idle_period(&q->vars);

        sch_tree_unlock(sch);

        red_offload(sch, true);

        if (old_child)
                qdisc_put(old_child);
        return 0;

unlock_out:
        sch_tree_unlock(sch);
        if (child)
                qdisc_put(child);
        return err;
}

static inline void red_adaptative_timer(struct timer_list *t)
{
        struct red_sched_data *q = from_timer(q, t, adapt_timer);
        struct Qdisc *sch = q->sch;
        spinlock_t *root_lock;

        rcu_read_lock();
        root_lock = qdisc_lock(qdisc_root_sleeping(sch));
        spin_lock(root_lock);
        red_adaptative_algo(&q->parms, &q->vars);
        mod_timer(&q->adapt_timer, jiffies + HZ/2);
        spin_unlock(root_lock);
        rcu_read_unlock();
}

static int red_init(struct Qdisc *sch, struct nlattr *opt,
                    struct netlink_ext_ack *extack)
{
        struct red_sched_data *q = qdisc_priv(sch);
        struct nlattr *tb[TCA_RED_MAX + 1];
        int err;

        q->qdisc = &noop_qdisc;
        q->sch = sch;
        timer_setup(&q->adapt_timer, red_adaptative_timer, 0);

        if (!opt)
                return -EINVAL;

        err = nla_parse_nested_deprecated(tb, TCA_RED_MAX, opt, red_policy,
                                          extack);
        if (err < 0)
                return err;

        err = __red_change(sch, tb, extack);
        if (err)
                return err;

        err = tcf_qevent_init(&q->qe_early_drop, sch,
                              FLOW_BLOCK_BINDER_TYPE_RED_EARLY_DROP,
                              tb[TCA_RED_EARLY_DROP_BLOCK], extack);
        if (err)
                return err;

        return tcf_qevent_init(&q->qe_mark, sch,
                               FLOW_BLOCK_BINDER_TYPE_RED_MARK,
                               tb[TCA_RED_MARK_BLOCK], extack);
}

static int red_change(struct Qdisc *sch, struct nlattr *opt,
                      struct netlink_ext_ack *extack)
{
        struct red_sched_data *q = qdisc_priv(sch);
        struct nlattr *tb[TCA_RED_MAX + 1];
        int err;

        err = nla_parse_nested_deprecated(tb, TCA_RED_MAX, opt, red_policy,
                                          extack);
        if (err < 0)
                return err;

        err = tcf_qevent_validate_change(&q->qe_early_drop,
                                         tb[TCA_RED_EARLY_DROP_BLOCK], extack);
        if (err)
                return err;

        err = tcf_qevent_validate_change(&q->qe_mark,
                                         tb[TCA_RED_MARK_BLOCK], extack);
        if (err)
                return err;

        return __red_change(sch, tb, extack);
}

static int red_dump_offload_stats(struct Qdisc *sch)
{
        struct tc_red_qopt_offload hw_stats = {
                .command = TC_RED_STATS,
                .handle = sch->handle,
                .parent = sch->parent,
                {
                        .stats.bstats = &sch->bstats,
                        .stats.qstats = &sch->qstats,
                },
        };

        return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_RED, &hw_stats);
}

static int red_dump(struct Qdisc *sch, struct sk_buff *skb)
{
        struct red_sched_data *q = qdisc_priv(sch);
        struct nlattr *opts = NULL;
        struct tc_red_qopt opt = {
                .limit                = q->limit,
                .flags                = (q->flags & TC_RED_HISTORIC_FLAGS) |
                                  q->userbits,
                .qth_min        = q->parms.qth_min >> q->parms.Wlog,
                .qth_max        = q->parms.qth_max >> q->parms.Wlog,
                .Wlog                = q->parms.Wlog,
                .Plog                = q->parms.Plog,
                .Scell_log        = q->parms.Scell_log,
        };
        int err;

        err = red_dump_offload_stats(sch);
        if (err)
                goto nla_put_failure;

        opts = nla_nest_start_noflag(skb, TCA_OPTIONS);
        if (opts == NULL)
                goto nla_put_failure;
        if (nla_put(skb, TCA_RED_PARMS, sizeof(opt), &opt) ||
            nla_put_u32(skb, TCA_RED_MAX_P, q->parms.max_P) ||
            nla_put_bitfield32(skb, TCA_RED_FLAGS,
                               q->flags, TC_RED_SUPPORTED_FLAGS) ||
            tcf_qevent_dump(skb, TCA_RED_MARK_BLOCK, &q->qe_mark) ||
            tcf_qevent_dump(skb, TCA_RED_EARLY_DROP_BLOCK, &q->qe_early_drop))
                goto nla_put_failure;
        return nla_nest_end(skb, opts);

nla_put_failure:
        nla_nest_cancel(skb, opts);
        return -EMSGSIZE;
}

static int red_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
{
        struct red_sched_data *q = qdisc_priv(sch);
        struct net_device *dev = qdisc_dev(sch);
        struct tc_red_xstats st = {0};

        if (sch->flags & TCQ_F_OFFLOADED) {
                struct tc_red_qopt_offload hw_stats_request = {
                        .command = TC_RED_XSTATS,
                        .handle = sch->handle,
                        .parent = sch->parent,
                        {
                                .xstats = &q->stats,
                        },
                };
                dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_RED,
                                              &hw_stats_request);
        }
        st.early = q->stats.prob_drop + q->stats.forced_drop;
        st.pdrop = q->stats.pdrop;
        st.marked = q->stats.prob_mark + q->stats.forced_mark;

        return gnet_stats_copy_app(d, &st, sizeof(st));
}

static int red_dump_class(struct Qdisc *sch, unsigned long cl,
                          struct sk_buff *skb, struct tcmsg *tcm)
{
        struct red_sched_data *q = qdisc_priv(sch);

        tcm->tcm_handle |= TC_H_MIN(1);
        tcm->tcm_info = q->qdisc->handle;
        return 0;
}

static void red_graft_offload(struct Qdisc *sch,
                              struct Qdisc *new, struct Qdisc *old,
                              struct netlink_ext_ack *extack)
{
        struct tc_red_qopt_offload graft_offload = {
                .handle                = sch->handle,
                .parent                = sch->parent,
                .child_handle        = new->handle,
                .command        = TC_RED_GRAFT,
        };

        qdisc_offload_graft_helper(qdisc_dev(sch), sch, new, old,
                                   TC_SETUP_QDISC_RED, &graft_offload, extack);
}

static int red_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
                     struct Qdisc **old, struct netlink_ext_ack *extack)
{
        struct red_sched_data *q = qdisc_priv(sch);

        if (new == NULL)
                new = &noop_qdisc;

        *old = qdisc_replace(sch, new, &q->qdisc);

        red_graft_offload(sch, new, *old, extack);
        return 0;
}

static struct Qdisc *red_leaf(struct Qdisc *sch, unsigned long arg)
{
        struct red_sched_data *q = qdisc_priv(sch);
        return q->qdisc;
}

static unsigned long red_find(struct Qdisc *sch, u32 classid)
{
        return 1;
}

static void red_walk(struct Qdisc *sch, struct qdisc_walker *walker)
{
        if (!walker->stop) {
                tc_qdisc_stats_dump(sch, 1, walker);
        }
}

static const struct Qdisc_class_ops red_class_ops = {
        .graft                =        red_graft,
        .leaf                =        red_leaf,
        .find                =        red_find,
        .walk                =        red_walk,
        .dump                =        red_dump_class,
};

static struct Qdisc_ops red_qdisc_ops __read_mostly = {
        .id                =        "red",
        .priv_size        =        sizeof(struct red_sched_data),
        .cl_ops                =        &red_class_ops,
        .enqueue        =        red_enqueue,
        .dequeue        =        red_dequeue,
        .peek                =        red_peek,
        .init                =        red_init,
        .reset                =        red_reset,
        .destroy        =        red_destroy,
        .change                =        red_change,
        .dump                =        red_dump,
        .dump_stats        =        red_dump_stats,
        .owner                =        THIS_MODULE,
};
MODULE_ALIAS_NET_SCH("red");

static int __init red_module_init(void)
{
        return register_qdisc(&red_qdisc_ops);
}

static void __exit red_module_exit(void)
{
        unregister_qdisc(&red_qdisc_ops);
}

module_init(red_module_init)
module_exit(red_module_exit)

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Random Early Detection qdisc");




























































































































































































































































































































































































































































































































































































































































    3 





    3 





























   10 





   10 





































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2020 Facebook */

#include <linux/fs.h>
#include <linux/anon_inodes.h>
#include <linux/filter.h>
#include <linux/bpf.h>
#include <linux/rcupdate_trace.h>

struct bpf_iter_target_info {
        struct list_head list;
        const struct bpf_iter_reg *reg_info;
        u32 btf_id;        /* cached value */
};

struct bpf_iter_link {
        struct bpf_link link;
        struct bpf_iter_aux_info aux;
        struct bpf_iter_target_info *tinfo;
};

struct bpf_iter_priv_data {
        struct bpf_iter_target_info *tinfo;
        const struct bpf_iter_seq_info *seq_info;
        struct bpf_prog *prog;
        u64 session_id;
        u64 seq_num;
        bool done_stop;
        u8 target_private[] __aligned(8);
};

static struct list_head targets = LIST_HEAD_INIT(targets);
static DEFINE_MUTEX(targets_mutex);

/* protect bpf_iter_link changes */
static DEFINE_MUTEX(link_mutex);

/* incremented on every opened seq_file */
static atomic64_t session_id;

static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
                            const struct bpf_iter_seq_info *seq_info);

static void bpf_iter_inc_seq_num(struct seq_file *seq)
{
        struct bpf_iter_priv_data *iter_priv;

        iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
                                 target_private);
        iter_priv->seq_num++;
}

static void bpf_iter_dec_seq_num(struct seq_file *seq)
{
        struct bpf_iter_priv_data *iter_priv;

        iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
                                 target_private);
        iter_priv->seq_num--;
}

static void bpf_iter_done_stop(struct seq_file *seq)
{
        struct bpf_iter_priv_data *iter_priv;

        iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
                                 target_private);
        iter_priv->done_stop = true;
}

static inline bool bpf_iter_target_support_resched(const struct bpf_iter_target_info *tinfo)
{
        return tinfo->reg_info->feature & BPF_ITER_RESCHED;
}

static bool bpf_iter_support_resched(struct seq_file *seq)
{
        struct bpf_iter_priv_data *iter_priv;

        iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
                                 target_private);
        return bpf_iter_target_support_resched(iter_priv->tinfo);
}

/* maximum visited objects before bailing out */
#define MAX_ITER_OBJECTS        1000000

/* bpf_seq_read, a customized and simpler version for bpf iterator.
 * The following are differences from seq_read():
 *  . fixed buffer size (PAGE_SIZE)
 *  . assuming NULL ->llseek()
 *  . stop() may call bpf program, handling potential overflow there
 */
static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size,
                            loff_t *ppos)
{
        struct seq_file *seq = file->private_data;
        size_t n, offs, copied = 0;
        int err = 0, num_objs = 0;
        bool can_resched;
        void *p;

        mutex_lock(&seq->lock);

        if (!seq->buf) {
                seq->size = PAGE_SIZE << 3;
                seq->buf = kvmalloc(seq->size, GFP_KERNEL);
                if (!seq->buf) {
                        err = -ENOMEM;
                        goto done;
                }
        }

        if (seq->count) {
                n = min(seq->count, size);
                err = copy_to_user(buf, seq->buf + seq->from, n);
                if (err) {
                        err = -EFAULT;
                        goto done;
                }
                seq->count -= n;
                seq->from += n;
                copied = n;
                goto done;
        }

        seq->from = 0;
        p = seq->op->start(seq, &seq->index);
        if (!p)
                goto stop;
        if (IS_ERR(p)) {
                err = PTR_ERR(p);
                seq->op->stop(seq, p);
                seq->count = 0;
                goto done;
        }

        err = seq->op->show(seq, p);
        if (err > 0) {
                /* object is skipped, decrease seq_num, so next
                 * valid object can reuse the same seq_num.
                 */
                bpf_iter_dec_seq_num(seq);
                seq->count = 0;
        } else if (err < 0 || seq_has_overflowed(seq)) {
                if (!err)
                        err = -E2BIG;
                seq->op->stop(seq, p);
                seq->count = 0;
                goto done;
        }

        can_resched = bpf_iter_support_resched(seq);
        while (1) {
                loff_t pos = seq->index;

                num_objs++;
                offs = seq->count;
                p = seq->op->next(seq, p, &seq->index);
                if (pos == seq->index) {
                        pr_info_ratelimited("buggy seq_file .next function %ps "
                                "did not updated position index\n",
                                seq->op->next);
                        seq->index++;
                }

                if (IS_ERR_OR_NULL(p))
                        break;

                /* got a valid next object, increase seq_num */
                bpf_iter_inc_seq_num(seq);

                if (seq->count >= size)
                        break;

                if (num_objs >= MAX_ITER_OBJECTS) {
                        if (offs == 0) {
                                err = -EAGAIN;
                                seq->op->stop(seq, p);
                                goto done;
                        }
                        break;
                }

                err = seq->op->show(seq, p);
                if (err > 0) {
                        bpf_iter_dec_seq_num(seq);
                        seq->count = offs;
                } else if (err < 0 || seq_has_overflowed(seq)) {
                        seq->count = offs;
                        if (offs == 0) {
                                if (!err)
                                        err = -E2BIG;
                                seq->op->stop(seq, p);
                                goto done;
                        }
                        break;
                }

                if (can_resched)
                        cond_resched();
        }
stop:
        offs = seq->count;
        if (IS_ERR(p)) {
                seq->op->stop(seq, NULL);
                err = PTR_ERR(p);
                goto done;
        }
        /* bpf program called if !p */
        seq->op->stop(seq, p);
        if (!p) {
                if (!seq_has_overflowed(seq)) {
                        bpf_iter_done_stop(seq);
                } else {
                        seq->count = offs;
                        if (offs == 0) {
                                err = -E2BIG;
                                goto done;
                        }
                }
        }

        n = min(seq->count, size);
        err = copy_to_user(buf, seq->buf, n);
        if (err) {
                err = -EFAULT;
                goto done;
        }
        copied = n;
        seq->count -= n;
        seq->from = n;
done:
        if (!copied)
                copied = err;
        else
                *ppos += copied;
        mutex_unlock(&seq->lock);
        return copied;
}

static const struct bpf_iter_seq_info *
__get_seq_info(struct bpf_iter_link *link)
{
        const struct bpf_iter_seq_info *seq_info;

        if (link->aux.map) {
                seq_info = link->aux.map->ops->iter_seq_info;
                if (seq_info)
                        return seq_info;
        }

        return link->tinfo->reg_info->seq_info;
}

static int iter_open(struct inode *inode, struct file *file)
{
        struct bpf_iter_link *link = inode->i_private;

        return prepare_seq_file(file, link, __get_seq_info(link));
}

static int iter_release(struct inode *inode, struct file *file)
{
        struct bpf_iter_priv_data *iter_priv;
        struct seq_file *seq;

        seq = file->private_data;
        if (!seq)
                return 0;

        iter_priv = container_of(seq->private, struct bpf_iter_priv_data,
                                 target_private);

        if (iter_priv->seq_info->fini_seq_private)
                iter_priv->seq_info->fini_seq_private(seq->private);

        bpf_prog_put(iter_priv->prog);
        seq->private = iter_priv;

        return seq_release_private(inode, file);
}

const struct file_operations bpf_iter_fops = {
        .open                = iter_open,
        .read                = bpf_seq_read,
        .release        = iter_release,
};

/* The argument reg_info will be cached in bpf_iter_target_info.
 * The common practice is to declare target reg_info as
 * a const static variable and passed as an argument to
 * bpf_iter_reg_target().
 */
int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info)
{
        struct bpf_iter_target_info *tinfo;

        tinfo = kzalloc(sizeof(*tinfo), GFP_KERNEL);
        if (!tinfo)
                return -ENOMEM;

        tinfo->reg_info = reg_info;
        INIT_LIST_HEAD(&tinfo->list);

        mutex_lock(&targets_mutex);
        list_add(&tinfo->list, &targets);
        mutex_unlock(&targets_mutex);

        return 0;
}

void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info)
{
        struct bpf_iter_target_info *tinfo;
        bool found = false;

        mutex_lock(&targets_mutex);
        list_for_each_entry(tinfo, &targets, list) {
                if (reg_info == tinfo->reg_info) {
                        list_del(&tinfo->list);
                        kfree(tinfo);
                        found = true;
                        break;
                }
        }
        mutex_unlock(&targets_mutex);

        WARN_ON(found == false);
}

static void cache_btf_id(struct bpf_iter_target_info *tinfo,
                         struct bpf_prog *prog)
{
        tinfo->btf_id = prog->aux->attach_btf_id;
}

bool bpf_iter_prog_supported(struct bpf_prog *prog)
{
        const char *attach_fname = prog->aux->attach_func_name;
        struct bpf_iter_target_info *tinfo = NULL, *iter;
        u32 prog_btf_id = prog->aux->attach_btf_id;
        const char *prefix = BPF_ITER_FUNC_PREFIX;
        int prefix_len = strlen(prefix);

        if (strncmp(attach_fname, prefix, prefix_len))
                return false;

        mutex_lock(&targets_mutex);
        list_for_each_entry(iter, &targets, list) {
                if (iter->btf_id && iter->btf_id == prog_btf_id) {
                        tinfo = iter;
                        break;
                }
                if (!strcmp(attach_fname + prefix_len, iter->reg_info->target)) {
                        cache_btf_id(iter, prog);
                        tinfo = iter;
                        break;
                }
        }
        mutex_unlock(&targets_mutex);

        if (tinfo) {
                prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size;
                prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info;
        }

        return tinfo != NULL;
}

const struct bpf_func_proto *
bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
{
        const struct bpf_iter_target_info *tinfo;
        const struct bpf_func_proto *fn = NULL;

        mutex_lock(&targets_mutex);
        list_for_each_entry(tinfo, &targets, list) {
                if (tinfo->btf_id == prog->aux->attach_btf_id) {
                        const struct bpf_iter_reg *reg_info;

                        reg_info = tinfo->reg_info;
                        if (reg_info->get_func_proto)
                                fn = reg_info->get_func_proto(func_id, prog);
                        break;
                }
        }
        mutex_unlock(&targets_mutex);

        return fn;
}

static void bpf_iter_link_release(struct bpf_link *link)
{
        struct bpf_iter_link *iter_link =
                container_of(link, struct bpf_iter_link, link);

        if (iter_link->tinfo->reg_info->detach_target)
                iter_link->tinfo->reg_info->detach_target(&iter_link->aux);
}

static void bpf_iter_link_dealloc(struct bpf_link *link)
{
        struct bpf_iter_link *iter_link =
                container_of(link, struct bpf_iter_link, link);

        kfree(iter_link);
}

static int bpf_iter_link_replace(struct bpf_link *link,
                                 struct bpf_prog *new_prog,
                                 struct bpf_prog *old_prog)
{
        int ret = 0;

        mutex_lock(&link_mutex);
        if (old_prog && link->prog != old_prog) {
                ret = -EPERM;
                goto out_unlock;
        }

        if (link->prog->type != new_prog->type ||
            link->prog->expected_attach_type != new_prog->expected_attach_type ||
            link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) {
                ret = -EINVAL;
                goto out_unlock;
        }

        old_prog = xchg(&link->prog, new_prog);
        bpf_prog_put(old_prog);

out_unlock:
        mutex_unlock(&link_mutex);
        return ret;
}

static void bpf_iter_link_show_fdinfo(const struct bpf_link *link,
                                      struct seq_file *seq)
{
        struct bpf_iter_link *iter_link =
                container_of(link, struct bpf_iter_link, link);
        bpf_iter_show_fdinfo_t show_fdinfo;

        seq_printf(seq,
                   "target_name:\t%s\n",
                   iter_link->tinfo->reg_info->target);

        show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo;
        if (show_fdinfo)
                show_fdinfo(&iter_link->aux, seq);
}

static int bpf_iter_link_fill_link_info(const struct bpf_link *link,
                                        struct bpf_link_info *info)
{
        struct bpf_iter_link *iter_link =
                container_of(link, struct bpf_iter_link, link);
        char __user *ubuf = u64_to_user_ptr(info->iter.target_name);
        bpf_iter_fill_link_info_t fill_link_info;
        u32 ulen = info->iter.target_name_len;
        const char *target_name;
        u32 target_len;

        if (!ulen ^ !ubuf)
                return -EINVAL;

        target_name = iter_link->tinfo->reg_info->target;
        target_len =  strlen(target_name);
        info->iter.target_name_len = target_len + 1;

        if (ubuf) {
                if (ulen >= target_len + 1) {
                        if (copy_to_user(ubuf, target_name, target_len + 1))
                                return -EFAULT;
                } else {
                        char zero = '\0';

                        if (copy_to_user(ubuf, target_name, ulen - 1))
                                return -EFAULT;
                        if (put_user(zero, ubuf + ulen - 1))
                                return -EFAULT;
                        return -ENOSPC;
                }
        }

        fill_link_info = iter_link->tinfo->reg_info->fill_link_info;
        if (fill_link_info)
                return fill_link_info(&iter_link->aux, info);

        return 0;
}

static const struct bpf_link_ops bpf_iter_link_lops = {
        .release = bpf_iter_link_release,
        .dealloc = bpf_iter_link_dealloc,
        .update_prog = bpf_iter_link_replace,
        .show_fdinfo = bpf_iter_link_show_fdinfo,
        .fill_link_info = bpf_iter_link_fill_link_info,
};

bool bpf_link_is_iter(struct bpf_link *link)
{
        return link->ops == &bpf_iter_link_lops;
}

int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr,
                         struct bpf_prog *prog)
{
        struct bpf_iter_target_info *tinfo = NULL, *iter;
        struct bpf_link_primer link_primer;
        union bpf_iter_link_info linfo;
        struct bpf_iter_link *link;
        u32 prog_btf_id, linfo_len;
        bpfptr_t ulinfo;
        int err;

        if (attr->link_create.target_fd || attr->link_create.flags)
                return -EINVAL;

        memset(&linfo, 0, sizeof(union bpf_iter_link_info));

        ulinfo = make_bpfptr(attr->link_create.iter_info, uattr.is_kernel);
        linfo_len = attr->link_create.iter_info_len;
        if (bpfptr_is_null(ulinfo) ^ !linfo_len)
                return -EINVAL;

        if (!bpfptr_is_null(ulinfo)) {
                err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo),
                                               linfo_len);
                if (err)
                        return err;
                linfo_len = min_t(u32, linfo_len, sizeof(linfo));
                if (copy_from_bpfptr(&linfo, ulinfo, linfo_len))
                        return -EFAULT;
        }

        prog_btf_id = prog->aux->attach_btf_id;
        mutex_lock(&targets_mutex);
        list_for_each_entry(iter, &targets, list) {
                if (iter->btf_id == prog_btf_id) {
                        tinfo = iter;
                        break;
                }
        }
        mutex_unlock(&targets_mutex);
        if (!tinfo)
                return -ENOENT;

        /* Only allow sleepable program for resched-able iterator */
        if (prog->sleepable && !bpf_iter_target_support_resched(tinfo))
                return -EINVAL;

        link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN);
        if (!link)
                return -ENOMEM;

        bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog);
        link->tinfo = tinfo;

        err = bpf_link_prime(&link->link, &link_primer);
        if (err) {
                kfree(link);
                return err;
        }

        if (tinfo->reg_info->attach_target) {
                err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux);
                if (err) {
                        bpf_link_cleanup(&link_primer);
                        return err;
                }
        }

        return bpf_link_settle(&link_primer);
}

static void init_seq_meta(struct bpf_iter_priv_data *priv_data,
                          struct bpf_iter_target_info *tinfo,
                          const struct bpf_iter_seq_info *seq_info,
                          struct bpf_prog *prog)
{
        priv_data->tinfo = tinfo;
        priv_data->seq_info = seq_info;
        priv_data->prog = prog;
        priv_data->session_id = atomic64_inc_return(&session_id);
        priv_data->seq_num = 0;
        priv_data->done_stop = false;
}

static int prepare_seq_file(struct file *file, struct bpf_iter_link *link,
                            const struct bpf_iter_seq_info *seq_info)
{
        struct bpf_iter_priv_data *priv_data;
        struct bpf_iter_target_info *tinfo;
        struct bpf_prog *prog;
        u32 total_priv_dsize;
        struct seq_file *seq;
        int err = 0;

        mutex_lock(&link_mutex);
        prog = link->link.prog;
        bpf_prog_inc(prog);
        mutex_unlock(&link_mutex);

        tinfo = link->tinfo;
        total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) +
                           seq_info->seq_priv_size;
        priv_data = __seq_open_private(file, seq_info->seq_ops,
                                       total_priv_dsize);
        if (!priv_data) {
                err = -ENOMEM;
                goto release_prog;
        }

        if (seq_info->init_seq_private) {
                err = seq_info->init_seq_private(priv_data->target_private, &link->aux);
                if (err)
                        goto release_seq_file;
        }

        init_seq_meta(priv_data, tinfo, seq_info, prog);
        seq = file->private_data;
        seq->private = priv_data->target_private;

        return 0;

release_seq_file:
        seq_release_private(file->f_inode, file);
        file->private_data = NULL;
release_prog:
        bpf_prog_put(prog);
        return err;
}

int bpf_iter_new_fd(struct bpf_link *link)
{
        struct bpf_iter_link *iter_link;
        struct file *file;
        unsigned int flags;
        int err, fd;

        if (link->ops != &bpf_iter_link_lops)
                return -EINVAL;

        flags = O_RDONLY | O_CLOEXEC;
        fd = get_unused_fd_flags(flags);
        if (fd < 0)
                return fd;

        file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags);
        if (IS_ERR(file)) {
                err = PTR_ERR(file);
                goto free_fd;
        }

        iter_link = container_of(link, struct bpf_iter_link, link);
        err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link));
        if (err)
                goto free_file;

        fd_install(fd, file);
        return fd;

free_file:
        fput(file);
free_fd:
        put_unused_fd(fd);
        return err;
}

struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop)
{
        struct bpf_iter_priv_data *iter_priv;
        struct seq_file *seq;
        void *seq_priv;

        seq = meta->seq;
        if (seq->file->f_op != &bpf_iter_fops)
                return NULL;

        seq_priv = seq->private;
        iter_priv = container_of(seq_priv, struct bpf_iter_priv_data,
                                 target_private);

        if (in_stop && iter_priv->done_stop)
                return NULL;

        meta->session_id = iter_priv->session_id;
        meta->seq_num = iter_priv->seq_num;

        return iter_priv->prog;
}

int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx)
{
        struct bpf_run_ctx run_ctx, *old_run_ctx;
        int ret;

        if (prog->sleepable) {
                rcu_read_lock_trace();
                migrate_disable();
                might_fault();
                old_run_ctx = bpf_set_run_ctx(&run_ctx);
                ret = bpf_prog_run(prog, ctx);
                bpf_reset_run_ctx(old_run_ctx);
                migrate_enable();
                rcu_read_unlock_trace();
        } else {
                rcu_read_lock();
                migrate_disable();
                old_run_ctx = bpf_set_run_ctx(&run_ctx);
                ret = bpf_prog_run(prog, ctx);
                bpf_reset_run_ctx(old_run_ctx);
                migrate_enable();
                rcu_read_unlock();
        }

        /* bpf program can only return 0 or 1:
         *  0 : okay
         *  1 : retry the same object
         * The bpf_iter_run_prog() return value
         * will be seq_ops->show() return value.
         */
        return ret == 0 ? 0 : -EAGAIN;
}

BPF_CALL_4(bpf_for_each_map_elem, struct bpf_map *, map, void *, callback_fn,
           void *, callback_ctx, u64, flags)
{
        return map->ops->map_for_each_callback(map, callback_fn, callback_ctx, flags);
}

const struct bpf_func_proto bpf_for_each_map_elem_proto = {
        .func                = bpf_for_each_map_elem,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_CONST_MAP_PTR,
        .arg2_type        = ARG_PTR_TO_FUNC,
        .arg3_type        = ARG_PTR_TO_STACK_OR_NULL,
        .arg4_type        = ARG_ANYTHING,
};

BPF_CALL_4(bpf_loop, u32, nr_loops, void *, callback_fn, void *, callback_ctx,
           u64, flags)
{
        bpf_callback_t callback = (bpf_callback_t)callback_fn;
        u64 ret;
        u32 i;

        /* Note: these safety checks are also verified when bpf_loop
         * is inlined, be careful to modify this code in sync. See
         * function verifier.c:inline_bpf_loop.
         */
        if (flags)
                return -EINVAL;
        if (nr_loops > BPF_MAX_LOOPS)
                return -E2BIG;

        for (i = 0; i < nr_loops; i++) {
                ret = callback((u64)i, (u64)(long)callback_ctx, 0, 0, 0);
                /* return value: 0 - continue, 1 - stop and return */
                if (ret)
                        return i + 1;
        }

        return i;
}

const struct bpf_func_proto bpf_loop_proto = {
        .func                = bpf_loop,
        .gpl_only        = false,
        .ret_type        = RET_INTEGER,
        .arg1_type        = ARG_ANYTHING,
        .arg2_type        = ARG_PTR_TO_FUNC,
        .arg3_type        = ARG_PTR_TO_STACK_OR_NULL,
        .arg4_type        = ARG_ANYTHING,
};

struct bpf_iter_num_kern {
        int cur; /* current value, inclusive */
        int end; /* final value, exclusive */
} __aligned(8);

__bpf_kfunc_start_defs();

__bpf_kfunc int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end)
{
        struct bpf_iter_num_kern *s = (void *)it;

        BUILD_BUG_ON(sizeof(struct bpf_iter_num_kern) != sizeof(struct bpf_iter_num));
        BUILD_BUG_ON(__alignof__(struct bpf_iter_num_kern) != __alignof__(struct bpf_iter_num));

        /* start == end is legit, it's an empty range and we'll just get NULL
         * on first (and any subsequent) bpf_iter_num_next() call
         */
        if (start > end) {
                s->cur = s->end = 0;
                return -EINVAL;
        }

        /* avoid overflows, e.g., if start == INT_MIN and end == INT_MAX */
        if ((s64)end - (s64)start > BPF_MAX_LOOPS) {
                s->cur = s->end = 0;
                return -E2BIG;
        }

        /* user will call bpf_iter_num_next() first,
         * which will set s->cur to exactly start value;
         * underflow shouldn't matter
         */
        s->cur = start - 1;
        s->end = end;

        return 0;
}

__bpf_kfunc int *bpf_iter_num_next(struct bpf_iter_num* it)
{
        struct bpf_iter_num_kern *s = (void *)it;

        /* check failed initialization or if we are done (same behavior);
         * need to be careful about overflow, so convert to s64 for checks,
         * e.g., if s->cur == s->end == INT_MAX, we can't just do
         * s->cur + 1 >= s->end
         */
        if ((s64)(s->cur + 1) >= s->end) {
                s->cur = s->end = 0;
                return NULL;
        }

        s->cur++;

        return &s->cur;
}

__bpf_kfunc void bpf_iter_num_destroy(struct bpf_iter_num *it)
{
        struct bpf_iter_num_kern *s = (void *)it;

        s->cur = s->end = 0;
}

__bpf_kfunc_end_defs();


























   16 







   17 



   16 




    4 








    1 








    3 

    1 








   13 







   13 
   13 
   13 




    2 



    2 


    2 
    1 






    2 







    2 


    2 













    1 
    2 



    2 
    1 









    1 




    1 


    1 




   11 




   11 

    1 




   10 


    1 










    1 


    1 



























    5 









    1 













    3 

    3 











    1 



    2 












    5 


    5 





















    1 




















    2 














    2 





    2 










   14 











   11 
    4 





   15 




    1 


   15 

   15 







   15 















   15 










   15 




   13 








   13 




    1 

    1 












   13 













    2 
    1 



    2 












   11 







   11 





























   11 
   11 
   11 































    3 

   11 





   10 








   11 








    2 


    1 

    1 




    2 


    1 

    1 




   11 


    1 

   10 




    2 


    1 

    1 




    2 


    1 

    1 




    4 


    1 

    3 




    9 


    9 

    2 



    3 



    1 




    1 




    1 




    1 





    2 








   20 



   19 


   20 

    1 


    1 




    3 

    8 
    8 







    8 












    8 
    8 














    8 






    2 

    6 
    6 
    6 


















    6 



    6 
    6 


































    3 


    1 


    2 
















    1 




    1 













    1 




    2 

















    1 


    1 





















































































    3 


    1 









    1 



    1 








    1 







    5 


    1 


    1 


    1 
    1 



    1 














    2 
    2 






    1 




    1 




    2 




    1 

    1 




    4 






    2 

    2 








    4 








    5 

    1 



    4 





    2 






    4 


    2 




    2 








    7 


    1 
    1 


    5 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
// SPDX-License-Identifier: GPL-2.0-only
/*
 * vivid-vid-out.c - video output support functions.
 *
 * Copyright 2014 Cisco Systems, Inc. and/or its affiliates. All rights reserved.
 */

#include <linux/errno.h>
#include <linux/kernel.h>
#include <linux/sched.h>
#include <linux/videodev2.h>
#include <linux/v4l2-dv-timings.h>
#include <media/v4l2-common.h>
#include <media/v4l2-event.h>
#include <media/v4l2-dv-timings.h>
#include <media/v4l2-rect.h>

#include "vivid-core.h"
#include "vivid-vid-common.h"
#include "vivid-kthread-out.h"
#include "vivid-vid-out.h"

static int vid_out_queue_setup(struct vb2_queue *vq,
                       unsigned *nbuffers, unsigned *nplanes,
                       unsigned sizes[], struct device *alloc_devs[])
{
        struct vivid_dev *dev = vb2_get_drv_priv(vq);
        const struct vivid_fmt *vfmt = dev->fmt_out;
        unsigned planes = vfmt->buffers;
        unsigned h = dev->fmt_out_rect.height;
        unsigned int size = dev->bytesperline_out[0] * h + vfmt->data_offset[0];
        unsigned p;

        for (p = vfmt->buffers; p < vfmt->planes; p++)
                size += dev->bytesperline_out[p] * h / vfmt->vdownsampling[p] +
                        vfmt->data_offset[p];

        if (dev->field_out == V4L2_FIELD_ALTERNATE) {
                /*
                 * You cannot use write() with FIELD_ALTERNATE since the field
                 * information (TOP/BOTTOM) cannot be passed to the kernel.
                 */
                if (vb2_fileio_is_active(vq))
                        return -EINVAL;
        }

        if (dev->queue_setup_error) {
                /*
                 * Error injection: test what happens if queue_setup() returns
                 * an error.
                 */
                dev->queue_setup_error = false;
                return -EINVAL;
        }

        if (*nplanes) {
                /*
                 * Check if the number of requested planes match
                 * the number of planes in the current format. You can't mix that.
                 */
                if (*nplanes != planes)
                        return -EINVAL;
                if (sizes[0] < size)
                        return -EINVAL;
                for (p = 1; p < planes; p++) {
                        if (sizes[p] < dev->bytesperline_out[p] * h /
                                        vfmt->vdownsampling[p] +
                                        vfmt->data_offset[p])
                                return -EINVAL;
                }
        } else {
                for (p = 0; p < planes; p++)
                        sizes[p] = p ? dev->bytesperline_out[p] * h /
                                        vfmt->vdownsampling[p] +
                                        vfmt->data_offset[p] : size;
        }

        *nplanes = planes;

        dprintk(dev, 1, "%s: count=%u\n", __func__, *nbuffers);
        for (p = 0; p < planes; p++)
                dprintk(dev, 1, "%s: size[%u]=%u\n", __func__, p, sizes[p]);
        return 0;
}

static int vid_out_buf_out_validate(struct vb2_buffer *vb)
{
        struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
        struct vivid_dev *dev = vb2_get_drv_priv(vb->vb2_queue);

        dprintk(dev, 1, "%s\n", __func__);

        if (dev->field_out != V4L2_FIELD_ALTERNATE)
                vbuf->field = dev->field_out;
        else if (vbuf->field != V4L2_FIELD_TOP &&
                 vbuf->field != V4L2_FIELD_BOTTOM)
                return -EINVAL;
        return 0;
}

static int vid_out_buf_prepare(struct vb2_buffer *vb)
{
        struct vivid_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
        const struct vivid_fmt *vfmt = dev->fmt_out;
        unsigned int planes = vfmt->buffers;
        unsigned int h = dev->fmt_out_rect.height;
        unsigned int size = dev->bytesperline_out[0] * h;
        unsigned p;

        for (p = vfmt->buffers; p < vfmt->planes; p++)
                size += dev->bytesperline_out[p] * h / vfmt->vdownsampling[p];

        dprintk(dev, 1, "%s\n", __func__);

        if (WARN_ON(NULL == dev->fmt_out))
                return -EINVAL;

        if (dev->buf_prepare_error) {
                /*
                 * Error injection: test what happens if buf_prepare() returns
                 * an error.
                 */
                dev->buf_prepare_error = false;
                return -EINVAL;
        }

        for (p = 0; p < planes; p++) {
                if (p)
                        size = dev->bytesperline_out[p] * h / vfmt->vdownsampling[p];
                size += vb->planes[p].data_offset;

                if (vb2_get_plane_payload(vb, p) < size) {
                        dprintk(dev, 1, "%s the payload is too small for plane %u (%lu < %u)\n",
                                        __func__, p, vb2_get_plane_payload(vb, p), size);
                        return -EINVAL;
                }
        }

        return 0;
}

static void vid_out_buf_queue(struct vb2_buffer *vb)
{
        struct vb2_v4l2_buffer *vbuf = to_vb2_v4l2_buffer(vb);
        struct vivid_dev *dev = vb2_get_drv_priv(vb->vb2_queue);
        struct vivid_buffer *buf = container_of(vbuf, struct vivid_buffer, vb);

        dprintk(dev, 1, "%s\n", __func__);

        spin_lock(&dev->slock);
        list_add_tail(&buf->list, &dev->vid_out_active);
        spin_unlock(&dev->slock);
}

static int vid_out_start_streaming(struct vb2_queue *vq, unsigned count)
{
        struct vivid_dev *dev = vb2_get_drv_priv(vq);
        int err;

        dev->vid_out_seq_count = 0;
        dprintk(dev, 1, "%s\n", __func__);
        if (dev->start_streaming_error) {
                dev->start_streaming_error = false;
                err = -EINVAL;
        } else {
                err = vivid_start_generating_vid_out(dev, &dev->vid_out_streaming);
        }
        if (err) {
                struct vivid_buffer *buf, *tmp;

                list_for_each_entry_safe(buf, tmp, &dev->vid_out_active, list) {
                        list_del(&buf->list);
                        vb2_buffer_done(&buf->vb.vb2_buf,
                                        VB2_BUF_STATE_QUEUED);
                }
        }
        return err;
}

/* abort streaming and wait for last buffer */
static void vid_out_stop_streaming(struct vb2_queue *vq)
{
        struct vivid_dev *dev = vb2_get_drv_priv(vq);

        dprintk(dev, 1, "%s\n", __func__);
        vivid_stop_generating_vid_out(dev, &dev->vid_out_streaming);
}

static void vid_out_buf_request_complete(struct vb2_buffer *vb)
{
        struct vivid_dev *dev = vb2_get_drv_priv(vb->vb2_queue);

        v4l2_ctrl_request_complete(vb->req_obj.req, &dev->ctrl_hdl_vid_out);
}

const struct vb2_ops vivid_vid_out_qops = {
        .queue_setup                = vid_out_queue_setup,
        .buf_out_validate                = vid_out_buf_out_validate,
        .buf_prepare                = vid_out_buf_prepare,
        .buf_queue                = vid_out_buf_queue,
        .start_streaming        = vid_out_start_streaming,
        .stop_streaming                = vid_out_stop_streaming,
        .buf_request_complete        = vid_out_buf_request_complete,
        .wait_prepare                = vb2_ops_wait_prepare,
        .wait_finish                = vb2_ops_wait_finish,
};

/*
 * Called whenever the format has to be reset which can occur when
 * changing outputs, standard, timings, etc.
 */
void vivid_update_format_out(struct vivid_dev *dev)
{
        struct v4l2_bt_timings *bt = &dev->dv_timings_out.bt;
        unsigned size, p;
        u64 pixelclock;

        switch (dev->output_type[dev->output]) {
        case SVID:
        default:
                dev->field_out = dev->tv_field_out;
                dev->sink_rect.width = 720;
                if (dev->std_out & V4L2_STD_525_60) {
                        dev->sink_rect.height = 480;
                        dev->timeperframe_vid_out = (struct v4l2_fract) { 1001, 30000 };
                        dev->service_set_out = V4L2_SLICED_CAPTION_525;
                } else {
                        dev->sink_rect.height = 576;
                        dev->timeperframe_vid_out = (struct v4l2_fract) { 1000, 25000 };
                        dev->service_set_out = V4L2_SLICED_WSS_625 | V4L2_SLICED_TELETEXT_B;
                }
                dev->colorspace_out = V4L2_COLORSPACE_SMPTE170M;
                break;
        case HDMI:
                dev->sink_rect.width = bt->width;
                dev->sink_rect.height = bt->height;
                size = V4L2_DV_BT_FRAME_WIDTH(bt) * V4L2_DV_BT_FRAME_HEIGHT(bt);

                if (can_reduce_fps(bt) && (bt->flags & V4L2_DV_FL_REDUCED_FPS))
                        pixelclock = div_u64(bt->pixelclock * 1000, 1001);
                else
                        pixelclock = bt->pixelclock;

                dev->timeperframe_vid_out = (struct v4l2_fract) {
                        size / 100, (u32)pixelclock / 100
                };
                if (bt->interlaced)
                        dev->field_out = V4L2_FIELD_ALTERNATE;
                else
                        dev->field_out = V4L2_FIELD_NONE;
                if (!dev->dvi_d_out && (bt->flags & V4L2_DV_FL_IS_CE_VIDEO)) {
                        if (bt->width == 720 && bt->height <= 576)
                                dev->colorspace_out = V4L2_COLORSPACE_SMPTE170M;
                        else
                                dev->colorspace_out = V4L2_COLORSPACE_REC709;
                } else {
                        dev->colorspace_out = V4L2_COLORSPACE_SRGB;
                }
                break;
        }
        dev->xfer_func_out = V4L2_XFER_FUNC_DEFAULT;
        dev->ycbcr_enc_out = V4L2_YCBCR_ENC_DEFAULT;
        dev->hsv_enc_out = V4L2_HSV_ENC_180;
        dev->quantization_out = V4L2_QUANTIZATION_DEFAULT;
        dev->compose_out = dev->sink_rect;
        dev->compose_bounds_out = dev->sink_rect;
        dev->crop_out = dev->compose_out;
        if (V4L2_FIELD_HAS_T_OR_B(dev->field_out))
                dev->crop_out.height /= 2;
        dev->fmt_out_rect = dev->crop_out;
        for (p = 0; p < dev->fmt_out->planes; p++)
                dev->bytesperline_out[p] =
                        (dev->sink_rect.width * dev->fmt_out->bit_depth[p]) / 8;
}

/* Map the field to something that is valid for the current output */
static enum v4l2_field vivid_field_out(struct vivid_dev *dev, enum v4l2_field field)
{
        if (vivid_is_svid_out(dev)) {
                switch (field) {
                case V4L2_FIELD_INTERLACED_TB:
                case V4L2_FIELD_INTERLACED_BT:
                case V4L2_FIELD_SEQ_TB:
                case V4L2_FIELD_SEQ_BT:
                case V4L2_FIELD_ALTERNATE:
                        return field;
                case V4L2_FIELD_INTERLACED:
                default:
                        return V4L2_FIELD_INTERLACED;
                }
        }
        if (vivid_is_hdmi_out(dev))
                return dev->dv_timings_out.bt.interlaced ? V4L2_FIELD_ALTERNATE :
                                                       V4L2_FIELD_NONE;
        return V4L2_FIELD_NONE;
}

static enum tpg_pixel_aspect vivid_get_pixel_aspect(const struct vivid_dev *dev)
{
        if (vivid_is_svid_out(dev))
                return (dev->std_out & V4L2_STD_525_60) ?
                        TPG_PIXEL_ASPECT_NTSC : TPG_PIXEL_ASPECT_PAL;

        if (vivid_is_hdmi_out(dev) &&
            dev->sink_rect.width == 720 && dev->sink_rect.height <= 576)
                return dev->sink_rect.height == 480 ?
                        TPG_PIXEL_ASPECT_NTSC : TPG_PIXEL_ASPECT_PAL;

        return TPG_PIXEL_ASPECT_SQUARE;
}

int vivid_g_fmt_vid_out(struct file *file, void *priv,
                                        struct v4l2_format *f)
{
        struct vivid_dev *dev = video_drvdata(file);
        struct v4l2_pix_format_mplane *mp = &f->fmt.pix_mp;
        const struct vivid_fmt *fmt = dev->fmt_out;
        unsigned p;

        mp->width        = dev->fmt_out_rect.width;
        mp->height       = dev->fmt_out_rect.height;
        mp->field        = dev->field_out;
        mp->pixelformat  = fmt->fourcc;
        mp->colorspace   = dev->colorspace_out;
        mp->xfer_func    = dev->xfer_func_out;
        mp->ycbcr_enc    = dev->ycbcr_enc_out;
        mp->quantization = dev->quantization_out;
        mp->num_planes = fmt->buffers;
        for (p = 0; p < mp->num_planes; p++) {
                mp->plane_fmt[p].bytesperline = dev->bytesperline_out[p];
                mp->plane_fmt[p].sizeimage =
                        mp->plane_fmt[p].bytesperline * mp->height /
                        fmt->vdownsampling[p] + fmt->data_offset[p];
        }
        for (p = fmt->buffers; p < fmt->planes; p++) {
                unsigned stride = dev->bytesperline_out[p];

                mp->plane_fmt[0].sizeimage +=
                        (stride * mp->height) / fmt->vdownsampling[p];
        }
        return 0;
}

int vivid_try_fmt_vid_out(struct file *file, void *priv,
                        struct v4l2_format *f)
{
        struct vivid_dev *dev = video_drvdata(file);
        struct v4l2_bt_timings *bt = &dev->dv_timings_out.bt;
        struct v4l2_pix_format_mplane *mp = &f->fmt.pix_mp;
        struct v4l2_plane_pix_format *pfmt = mp->plane_fmt;
        const struct vivid_fmt *fmt;
        unsigned bytesperline, max_bpl;
        unsigned factor = 1;
        unsigned w, h;
        unsigned p;

        fmt = vivid_get_format(dev, mp->pixelformat);
        if (!fmt) {
                dprintk(dev, 1, "Fourcc format (0x%08x) unknown.\n",
                        mp->pixelformat);
                mp->pixelformat = V4L2_PIX_FMT_YUYV;
                fmt = vivid_get_format(dev, mp->pixelformat);
        }

        mp->field = vivid_field_out(dev, mp->field);
        if (vivid_is_svid_out(dev)) {
                w = 720;
                h = (dev->std_out & V4L2_STD_525_60) ? 480 : 576;
        } else {
                w = dev->sink_rect.width;
                h = dev->sink_rect.height;
        }
        if (V4L2_FIELD_HAS_T_OR_B(mp->field))
                factor = 2;
        if (!dev->has_scaler_out && !dev->has_crop_out && !dev->has_compose_out) {
                mp->width = w;
                mp->height = h / factor;
        } else {
                struct v4l2_rect r = { 0, 0, mp->width, mp->height * factor };

                v4l2_rect_set_min_size(&r, &vivid_min_rect);
                v4l2_rect_set_max_size(&r, &vivid_max_rect);
                if (dev->has_scaler_out && !dev->has_crop_out) {
                        struct v4l2_rect max_r = { 0, 0, MAX_ZOOM * w, MAX_ZOOM * h };

                        v4l2_rect_set_max_size(&r, &max_r);
                } else if (!dev->has_scaler_out && dev->has_compose_out && !dev->has_crop_out) {
                        v4l2_rect_set_max_size(&r, &dev->sink_rect);
                } else if (!dev->has_scaler_out && !dev->has_compose_out) {
                        v4l2_rect_set_min_size(&r, &dev->sink_rect);
                }
                mp->width = r.width;
                mp->height = r.height / factor;
        }

        /* This driver supports custom bytesperline values */

        mp->num_planes = fmt->buffers;
        for (p = 0; p < fmt->buffers; p++) {
                /* Calculate the minimum supported bytesperline value */
                bytesperline = (mp->width * fmt->bit_depth[p]) >> 3;
                /* Calculate the maximum supported bytesperline value */
                max_bpl = (MAX_ZOOM * MAX_WIDTH * fmt->bit_depth[p]) >> 3;

                if (pfmt[p].bytesperline > max_bpl)
                        pfmt[p].bytesperline = max_bpl;
                if (pfmt[p].bytesperline < bytesperline)
                        pfmt[p].bytesperline = bytesperline;

                pfmt[p].sizeimage = (pfmt[p].bytesperline * mp->height) /
                                fmt->vdownsampling[p] + fmt->data_offset[p];

                memset(pfmt[p].reserved, 0, sizeof(pfmt[p].reserved));
        }
        for (p = fmt->buffers; p < fmt->planes; p++)
                pfmt[0].sizeimage += (pfmt[0].bytesperline * mp->height *
                        (fmt->bit_depth[p] / fmt->vdownsampling[p])) /
                        (fmt->bit_depth[0] / fmt->vdownsampling[0]);

        mp->xfer_func = V4L2_XFER_FUNC_DEFAULT;
        mp->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
        mp->quantization = V4L2_QUANTIZATION_DEFAULT;
        if (vivid_is_svid_out(dev)) {
                mp->colorspace = V4L2_COLORSPACE_SMPTE170M;
        } else if (dev->dvi_d_out || !(bt->flags & V4L2_DV_FL_IS_CE_VIDEO)) {
                mp->colorspace = V4L2_COLORSPACE_SRGB;
                if (dev->dvi_d_out)
                        mp->quantization = V4L2_QUANTIZATION_LIM_RANGE;
        } else if (bt->width == 720 && bt->height <= 576) {
                mp->colorspace = V4L2_COLORSPACE_SMPTE170M;
        } else if (mp->colorspace != V4L2_COLORSPACE_SMPTE170M &&
                   mp->colorspace != V4L2_COLORSPACE_REC709 &&
                   mp->colorspace != V4L2_COLORSPACE_OPRGB &&
                   mp->colorspace != V4L2_COLORSPACE_BT2020 &&
                   mp->colorspace != V4L2_COLORSPACE_SRGB) {
                mp->colorspace = V4L2_COLORSPACE_REC709;
        }
        memset(mp->reserved, 0, sizeof(mp->reserved));
        return 0;
}

int vivid_s_fmt_vid_out(struct file *file, void *priv,
                                        struct v4l2_format *f)
{
        struct v4l2_pix_format_mplane *mp = &f->fmt.pix_mp;
        struct vivid_dev *dev = video_drvdata(file);
        struct v4l2_rect *crop = &dev->crop_out;
        struct v4l2_rect *compose = &dev->compose_out;
        struct vb2_queue *q = &dev->vb_vid_out_q;
        int ret = vivid_try_fmt_vid_out(file, priv, f);
        unsigned factor = 1;
        unsigned p;

        if (ret < 0)
                return ret;

        if (vb2_is_busy(q) &&
            (vivid_is_svid_out(dev) ||
             mp->width != dev->fmt_out_rect.width ||
             mp->height != dev->fmt_out_rect.height ||
             mp->pixelformat != dev->fmt_out->fourcc ||
             mp->field != dev->field_out)) {
                dprintk(dev, 1, "%s device busy\n", __func__);
                return -EBUSY;
        }

        /*
         * Allow for changing the colorspace on the fly. Useful for testing
         * purposes, and it is something that HDMI transmitters are able
         * to do.
         */
        if (vb2_is_busy(q))
                goto set_colorspace;

        dev->fmt_out = vivid_get_format(dev, mp->pixelformat);
        if (V4L2_FIELD_HAS_T_OR_B(mp->field))
                factor = 2;

        if (dev->has_scaler_out || dev->has_crop_out || dev->has_compose_out) {
                struct v4l2_rect r = { 0, 0, mp->width, mp->height };

                if (dev->has_scaler_out) {
                        if (dev->has_crop_out)
                                v4l2_rect_map_inside(crop, &r);
                        else
                                *crop = r;
                        if (dev->has_compose_out && !dev->has_crop_out) {
                                struct v4l2_rect min_r = {
                                        0, 0,
                                        r.width / MAX_ZOOM,
                                        factor * r.height / MAX_ZOOM
                                };
                                struct v4l2_rect max_r = {
                                        0, 0,
                                        r.width * MAX_ZOOM,
                                        factor * r.height * MAX_ZOOM
                                };

                                v4l2_rect_set_min_size(compose, &min_r);
                                v4l2_rect_set_max_size(compose, &max_r);
                                v4l2_rect_map_inside(compose, &dev->compose_bounds_out);
                        } else if (dev->has_compose_out) {
                                struct v4l2_rect min_r = {
                                        0, 0,
                                        crop->width / MAX_ZOOM,
                                        factor * crop->height / MAX_ZOOM
                                };
                                struct v4l2_rect max_r = {
                                        0, 0,
                                        crop->width * MAX_ZOOM,
                                        factor * crop->height * MAX_ZOOM
                                };

                                v4l2_rect_set_min_size(compose, &min_r);
                                v4l2_rect_set_max_size(compose, &max_r);
                                v4l2_rect_map_inside(compose, &dev->compose_bounds_out);
                        }
                } else if (dev->has_compose_out && !dev->has_crop_out) {
                        v4l2_rect_set_size_to(crop, &r);
                        r.height *= factor;
                        v4l2_rect_set_size_to(compose, &r);
                        v4l2_rect_map_inside(compose, &dev->compose_bounds_out);
                } else if (!dev->has_compose_out) {
                        v4l2_rect_map_inside(crop, &r);
                        r.height /= factor;
                        v4l2_rect_set_size_to(compose, &r);
                } else {
                        r.height *= factor;
                        v4l2_rect_set_max_size(compose, &r);
                        v4l2_rect_map_inside(compose, &dev->compose_bounds_out);
                        crop->top *= factor;
                        crop->height *= factor;
                        v4l2_rect_set_size_to(crop, compose);
                        v4l2_rect_map_inside(crop, &r);
                        crop->top /= factor;
                        crop->height /= factor;
                }
        } else {
                struct v4l2_rect r = { 0, 0, mp->width, mp->height };

                v4l2_rect_set_size_to(crop, &r);
                r.height /= factor;
                v4l2_rect_set_size_to(compose, &r);
        }

        dev->fmt_out_rect.width = mp->width;
        dev->fmt_out_rect.height = mp->height;
        for (p = 0; p < mp->num_planes; p++)
                dev->bytesperline_out[p] = mp->plane_fmt[p].bytesperline;
        for (p = dev->fmt_out->buffers; p < dev->fmt_out->planes; p++)
                dev->bytesperline_out[p] =
                        (dev->bytesperline_out[0] * dev->fmt_out->bit_depth[p]) /
                        dev->fmt_out->bit_depth[0];
        dev->field_out = mp->field;
        if (vivid_is_svid_out(dev))
                dev->tv_field_out = mp->field;

set_colorspace:
        dev->colorspace_out = mp->colorspace;
        dev->xfer_func_out = mp->xfer_func;
        dev->ycbcr_enc_out = mp->ycbcr_enc;
        dev->quantization_out = mp->quantization;
        struct vivid_dev *in_dev = vivid_output_is_connected_to(dev);

        if (in_dev) {
                vivid_send_source_change(in_dev, SVID);
                vivid_send_source_change(in_dev, HDMI);
        }
        return 0;
}

int vidioc_g_fmt_vid_out_mplane(struct file *file, void *priv,
                                        struct v4l2_format *f)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (!dev->multiplanar)
                return -ENOTTY;
        return vivid_g_fmt_vid_out(file, priv, f);
}

int vidioc_try_fmt_vid_out_mplane(struct file *file, void *priv,
                        struct v4l2_format *f)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (!dev->multiplanar)
                return -ENOTTY;
        return vivid_try_fmt_vid_out(file, priv, f);
}

int vidioc_s_fmt_vid_out_mplane(struct file *file, void *priv,
                        struct v4l2_format *f)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (!dev->multiplanar)
                return -ENOTTY;
        return vivid_s_fmt_vid_out(file, priv, f);
}

int vidioc_g_fmt_vid_out(struct file *file, void *priv,
                                        struct v4l2_format *f)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (dev->multiplanar)
                return -ENOTTY;
        return fmt_sp2mp_func(file, priv, f, vivid_g_fmt_vid_out);
}

int vidioc_try_fmt_vid_out(struct file *file, void *priv,
                        struct v4l2_format *f)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (dev->multiplanar)
                return -ENOTTY;
        return fmt_sp2mp_func(file, priv, f, vivid_try_fmt_vid_out);
}

int vidioc_s_fmt_vid_out(struct file *file, void *priv,
                        struct v4l2_format *f)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (dev->multiplanar)
                return -ENOTTY;
        return fmt_sp2mp_func(file, priv, f, vivid_s_fmt_vid_out);
}

int vivid_vid_out_g_selection(struct file *file, void *priv,
                              struct v4l2_selection *sel)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (!dev->has_crop_out && !dev->has_compose_out)
                return -ENOTTY;
        if (sel->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
                return -EINVAL;

        sel->r.left = sel->r.top = 0;
        switch (sel->target) {
        case V4L2_SEL_TGT_CROP:
                if (!dev->has_crop_out)
                        return -EINVAL;
                sel->r = dev->crop_out;
                break;
        case V4L2_SEL_TGT_CROP_DEFAULT:
                if (!dev->has_crop_out)
                        return -EINVAL;
                sel->r = dev->fmt_out_rect;
                break;
        case V4L2_SEL_TGT_CROP_BOUNDS:
                if (!dev->has_crop_out)
                        return -EINVAL;
                sel->r = vivid_max_rect;
                break;
        case V4L2_SEL_TGT_COMPOSE:
                if (!dev->has_compose_out)
                        return -EINVAL;
                sel->r = dev->compose_out;
                break;
        case V4L2_SEL_TGT_COMPOSE_DEFAULT:
        case V4L2_SEL_TGT_COMPOSE_BOUNDS:
                if (!dev->has_compose_out)
                        return -EINVAL;
                sel->r = dev->sink_rect;
                break;
        default:
                return -EINVAL;
        }
        return 0;
}

int vivid_vid_out_s_selection(struct file *file, void *fh, struct v4l2_selection *s)
{
        struct vivid_dev *dev = video_drvdata(file);
        struct v4l2_rect *crop = &dev->crop_out;
        struct v4l2_rect *compose = &dev->compose_out;
        unsigned factor = V4L2_FIELD_HAS_T_OR_B(dev->field_out) ? 2 : 1;
        int ret;

        if (!dev->has_crop_out && !dev->has_compose_out)
                return -ENOTTY;
        if (s->type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
                return -EINVAL;

        switch (s->target) {
        case V4L2_SEL_TGT_CROP:
                if (!dev->has_crop_out)
                        return -EINVAL;
                ret = vivid_vid_adjust_sel(s->flags, &s->r);
                if (ret)
                        return ret;
                v4l2_rect_set_min_size(&s->r, &vivid_min_rect);
                v4l2_rect_set_max_size(&s->r, &dev->fmt_out_rect);
                if (dev->has_scaler_out) {
                        struct v4l2_rect max_rect = {
                                0, 0,
                                dev->sink_rect.width * MAX_ZOOM,
                                (dev->sink_rect.height / factor) * MAX_ZOOM
                        };

                        v4l2_rect_set_max_size(&s->r, &max_rect);
                        if (dev->has_compose_out) {
                                struct v4l2_rect min_rect = {
                                        0, 0,
                                        s->r.width / MAX_ZOOM,
                                        (s->r.height * factor) / MAX_ZOOM
                                };
                                struct v4l2_rect max_rect = {
                                        0, 0,
                                        s->r.width * MAX_ZOOM,
                                        (s->r.height * factor) * MAX_ZOOM
                                };

                                v4l2_rect_set_min_size(compose, &min_rect);
                                v4l2_rect_set_max_size(compose, &max_rect);
                                v4l2_rect_map_inside(compose, &dev->compose_bounds_out);
                        }
                } else if (dev->has_compose_out) {
                        s->r.top *= factor;
                        s->r.height *= factor;
                        v4l2_rect_set_max_size(&s->r, &dev->sink_rect);
                        v4l2_rect_set_size_to(compose, &s->r);
                        v4l2_rect_map_inside(compose, &dev->compose_bounds_out);
                        s->r.top /= factor;
                        s->r.height /= factor;
                } else {
                        v4l2_rect_set_size_to(&s->r, &dev->sink_rect);
                        s->r.height /= factor;
                }
                v4l2_rect_map_inside(&s->r, &dev->fmt_out_rect);
                *crop = s->r;
                break;
        case V4L2_SEL_TGT_COMPOSE:
                if (!dev->has_compose_out)
                        return -EINVAL;
                ret = vivid_vid_adjust_sel(s->flags, &s->r);
                if (ret)
                        return ret;
                v4l2_rect_set_min_size(&s->r, &vivid_min_rect);
                v4l2_rect_set_max_size(&s->r, &dev->sink_rect);
                v4l2_rect_map_inside(&s->r, &dev->compose_bounds_out);
                s->r.top /= factor;
                s->r.height /= factor;
                if (dev->has_scaler_out) {
                        struct v4l2_rect fmt = dev->fmt_out_rect;
                        struct v4l2_rect max_rect = {
                                0, 0,
                                s->r.width * MAX_ZOOM,
                                s->r.height * MAX_ZOOM
                        };
                        struct v4l2_rect min_rect = {
                                0, 0,
                                s->r.width / MAX_ZOOM,
                                s->r.height / MAX_ZOOM
                        };

                        v4l2_rect_set_min_size(&fmt, &min_rect);
                        if (!dev->has_crop_out)
                                v4l2_rect_set_max_size(&fmt, &max_rect);
                        if (!v4l2_rect_same_size(&dev->fmt_out_rect, &fmt) &&
                            vb2_is_busy(&dev->vb_vid_out_q))
                                return -EBUSY;
                        if (dev->has_crop_out) {
                                v4l2_rect_set_min_size(crop, &min_rect);
                                v4l2_rect_set_max_size(crop, &max_rect);
                        }
                        dev->fmt_out_rect = fmt;
                } else if (dev->has_crop_out) {
                        struct v4l2_rect fmt = dev->fmt_out_rect;

                        v4l2_rect_set_min_size(&fmt, &s->r);
                        if (!v4l2_rect_same_size(&dev->fmt_out_rect, &fmt) &&
                            vb2_is_busy(&dev->vb_vid_out_q))
                                return -EBUSY;
                        dev->fmt_out_rect = fmt;
                        v4l2_rect_set_size_to(crop, &s->r);
                        v4l2_rect_map_inside(crop, &dev->fmt_out_rect);
                } else {
                        if (!v4l2_rect_same_size(&s->r, &dev->fmt_out_rect) &&
                            vb2_is_busy(&dev->vb_vid_out_q))
                                return -EBUSY;
                        v4l2_rect_set_size_to(&dev->fmt_out_rect, &s->r);
                        v4l2_rect_set_size_to(crop, &s->r);
                        crop->height /= factor;
                        v4l2_rect_map_inside(crop, &dev->fmt_out_rect);
                }
                s->r.top *= factor;
                s->r.height *= factor;
                *compose = s->r;
                break;
        default:
                return -EINVAL;
        }

        return 0;
}

int vivid_vid_out_g_pixelaspect(struct file *file, void *priv,
                                int type, struct v4l2_fract *f)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (type != V4L2_BUF_TYPE_VIDEO_OUTPUT)
                return -EINVAL;

        switch (vivid_get_pixel_aspect(dev)) {
        case TPG_PIXEL_ASPECT_NTSC:
                f->numerator = 11;
                f->denominator = 10;
                break;
        case TPG_PIXEL_ASPECT_PAL:
                f->numerator = 54;
                f->denominator = 59;
                break;
        default:
                break;
        }
        return 0;
}

int vidioc_g_fmt_vid_out_overlay(struct file *file, void *priv,
                                        struct v4l2_format *f)
{
        struct vivid_dev *dev = video_drvdata(file);
        const struct v4l2_rect *compose = &dev->compose_out;
        struct v4l2_window *win = &f->fmt.win;

        if (!dev->has_fb)
                return -EINVAL;
        win->w.top = dev->overlay_out_top;
        win->w.left = dev->overlay_out_left;
        win->w.width = compose->width;
        win->w.height = compose->height;
        win->field = V4L2_FIELD_ANY;
        win->chromakey = dev->chromakey_out;
        win->global_alpha = dev->global_alpha_out;
        return 0;
}

int vidioc_try_fmt_vid_out_overlay(struct file *file, void *priv,
                                        struct v4l2_format *f)
{
        struct vivid_dev *dev = video_drvdata(file);
        const struct v4l2_rect *compose = &dev->compose_out;
        struct v4l2_window *win = &f->fmt.win;

        if (!dev->has_fb)
                return -EINVAL;
        win->w.left = clamp_t(int, win->w.left,
                              -dev->display_width, dev->display_width);
        win->w.top = clamp_t(int, win->w.top,
                             -dev->display_height, dev->display_height);
        win->w.width = compose->width;
        win->w.height = compose->height;
        /*
         * It makes no sense for an OSD to overlay only top or bottom fields,
         * so always set this to ANY.
         */
        win->field = V4L2_FIELD_ANY;
        return 0;
}

int vidioc_s_fmt_vid_out_overlay(struct file *file, void *priv,
                                        struct v4l2_format *f)
{
        struct vivid_dev *dev = video_drvdata(file);
        struct v4l2_window *win = &f->fmt.win;
        int ret = vidioc_try_fmt_vid_out_overlay(file, priv, f);

        if (ret)
                return ret;

        dev->overlay_out_top = win->w.top;
        dev->overlay_out_left = win->w.left;
        dev->chromakey_out = win->chromakey;
        dev->global_alpha_out = win->global_alpha;
        return ret;
}

int vivid_vid_out_overlay(struct file *file, void *fh, unsigned i)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (i && !dev->fmt_out->can_do_overlay) {
                dprintk(dev, 1, "unsupported output format for output overlay\n");
                return -EINVAL;
        }

        dev->overlay_out_enabled = i;
        return 0;
}

int vivid_vid_out_g_fbuf(struct file *file, void *fh,
                                struct v4l2_framebuffer *a)
{
        struct vivid_dev *dev = video_drvdata(file);

        a->capability = V4L2_FBUF_CAP_EXTERNOVERLAY |
                        V4L2_FBUF_CAP_CHROMAKEY |
                        V4L2_FBUF_CAP_SRC_CHROMAKEY |
                        V4L2_FBUF_CAP_GLOBAL_ALPHA |
                        V4L2_FBUF_CAP_LOCAL_ALPHA |
                        V4L2_FBUF_CAP_LOCAL_INV_ALPHA;
        a->flags = V4L2_FBUF_FLAG_OVERLAY | dev->fbuf_out_flags;
        a->base = (void *)dev->video_pbase;
        a->fmt.width = dev->display_width;
        a->fmt.height = dev->display_height;
        if (dev->fb_defined.green.length == 5)
                a->fmt.pixelformat = V4L2_PIX_FMT_ARGB555;
        else
                a->fmt.pixelformat = V4L2_PIX_FMT_RGB565;
        a->fmt.bytesperline = dev->display_byte_stride;
        a->fmt.sizeimage = a->fmt.height * a->fmt.bytesperline;
        a->fmt.field = V4L2_FIELD_NONE;
        a->fmt.colorspace = V4L2_COLORSPACE_SRGB;
        a->fmt.priv = 0;
        return 0;
}

int vivid_vid_out_s_fbuf(struct file *file, void *fh,
                                const struct v4l2_framebuffer *a)
{
        struct vivid_dev *dev = video_drvdata(file);
        const unsigned chroma_flags = V4L2_FBUF_FLAG_CHROMAKEY |
                                      V4L2_FBUF_FLAG_SRC_CHROMAKEY;
        const unsigned alpha_flags = V4L2_FBUF_FLAG_GLOBAL_ALPHA |
                                     V4L2_FBUF_FLAG_LOCAL_ALPHA |
                                     V4L2_FBUF_FLAG_LOCAL_INV_ALPHA;


        if ((a->flags & chroma_flags) == chroma_flags)
                return -EINVAL;
        switch (a->flags & alpha_flags) {
        case 0:
        case V4L2_FBUF_FLAG_GLOBAL_ALPHA:
        case V4L2_FBUF_FLAG_LOCAL_ALPHA:
        case V4L2_FBUF_FLAG_LOCAL_INV_ALPHA:
                break;
        default:
                return -EINVAL;
        }
        dev->fbuf_out_flags &= ~(chroma_flags | alpha_flags);
        dev->fbuf_out_flags |= a->flags & (chroma_flags | alpha_flags);
        return 0;
}

static const struct v4l2_audioout vivid_audio_outputs[] = {
        { 0, "Line-Out 1" },
        { 1, "Line-Out 2" },
};

int vidioc_enum_output(struct file *file, void *priv,
                                struct v4l2_output *out)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (out->index >= dev->num_outputs)
                return -EINVAL;

        out->type = V4L2_OUTPUT_TYPE_ANALOG;
        switch (dev->output_type[out->index]) {
        case SVID:
                snprintf(out->name, sizeof(out->name), "S-Video %03u-%u",
                         dev->inst, dev->output_name_counter[out->index]);
                out->std = V4L2_STD_ALL;
                if (dev->has_audio_outputs)
                        out->audioset = (1 << ARRAY_SIZE(vivid_audio_outputs)) - 1;
                out->capabilities = V4L2_OUT_CAP_STD;
                break;
        case HDMI:
                snprintf(out->name, sizeof(out->name), "HDMI %03u-%u",
                         dev->inst, dev->output_name_counter[out->index]);
                out->capabilities = V4L2_OUT_CAP_DV_TIMINGS;
                break;
        }
        return 0;
}

int vidioc_g_output(struct file *file, void *priv, unsigned *o)
{
        struct vivid_dev *dev = video_drvdata(file);

        *o = dev->output;
        return 0;
}

int vidioc_s_output(struct file *file, void *priv, unsigned o)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (o >= dev->num_outputs)
                return -EINVAL;

        if (o == dev->output)
                return 0;

        if (vb2_is_busy(&dev->vb_vid_out_q) ||
            vb2_is_busy(&dev->vb_vbi_out_q) ||
            vb2_is_busy(&dev->vb_meta_out_q))
                return -EBUSY;

        dev->output = o;
        dev->tv_audio_output = 0;
        if (dev->output_type[o] == SVID)
                dev->vid_out_dev.tvnorms = V4L2_STD_ALL;
        else
                dev->vid_out_dev.tvnorms = 0;

        dev->vbi_out_dev.tvnorms = dev->vid_out_dev.tvnorms;
        dev->meta_out_dev.tvnorms = dev->vid_out_dev.tvnorms;
        vivid_update_format_out(dev);

        return 0;
}

int vidioc_enumaudout(struct file *file, void *fh, struct v4l2_audioout *vout)
{
        if (vout->index >= ARRAY_SIZE(vivid_audio_outputs))
                return -EINVAL;
        *vout = vivid_audio_outputs[vout->index];
        return 0;
}

int vidioc_g_audout(struct file *file, void *fh, struct v4l2_audioout *vout)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (!vivid_is_svid_out(dev))
                return -EINVAL;
        *vout = vivid_audio_outputs[dev->tv_audio_output];
        return 0;
}

int vidioc_s_audout(struct file *file, void *fh, const struct v4l2_audioout *vout)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (!vivid_is_svid_out(dev))
                return -EINVAL;
        if (vout->index >= ARRAY_SIZE(vivid_audio_outputs))
                return -EINVAL;
        dev->tv_audio_output = vout->index;
        return 0;
}

int vivid_vid_out_s_std(struct file *file, void *priv, v4l2_std_id id)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (!vivid_is_svid_out(dev))
                return -ENODATA;
        if (dev->std_out == id)
                return 0;
        if (vb2_is_busy(&dev->vb_vid_out_q) || vb2_is_busy(&dev->vb_vbi_out_q))
                return -EBUSY;
        dev->std_out = id;
        vivid_update_format_out(dev);
        return 0;
}

static bool valid_cvt_gtf_timings(struct v4l2_dv_timings *timings)
{
        struct v4l2_bt_timings *bt = &timings->bt;

        if ((bt->standards & (V4L2_DV_BT_STD_CVT | V4L2_DV_BT_STD_GTF)) &&
            v4l2_valid_dv_timings(timings, &vivid_dv_timings_cap, NULL, NULL))
                return true;

        return false;
}

int vivid_vid_out_s_dv_timings(struct file *file, void *_fh,
                                    struct v4l2_dv_timings *timings)
{
        struct vivid_dev *dev = video_drvdata(file);
        if (!vivid_is_hdmi_out(dev))
                return -ENODATA;
        if (!v4l2_find_dv_timings_cap(timings, &vivid_dv_timings_cap,
                                0, NULL, NULL) &&
            !valid_cvt_gtf_timings(timings))
                return -EINVAL;
        if (v4l2_match_dv_timings(timings, &dev->dv_timings_out, 0, true))
                return 0;
        if (vb2_is_busy(&dev->vb_vid_out_q))
                return -EBUSY;
        dev->dv_timings_out = *timings;
        vivid_update_format_out(dev);
        return 0;
}

int vivid_vid_out_g_parm(struct file *file, void *priv,
                          struct v4l2_streamparm *parm)
{
        struct vivid_dev *dev = video_drvdata(file);

        if (parm->type != (dev->multiplanar ?
                           V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE :
                           V4L2_BUF_TYPE_VIDEO_OUTPUT))
                return -EINVAL;

        parm->parm.output.capability   = V4L2_CAP_TIMEPERFRAME;
        parm->parm.output.timeperframe = dev->timeperframe_vid_out;
        parm->parm.output.writebuffers  = 1;

        return 0;
}

int vidioc_subscribe_event(struct v4l2_fh *fh,
                        const struct v4l2_event_subscription *sub)
{
        switch (sub->type) {
        case V4L2_EVENT_SOURCE_CHANGE:
                if (fh->vdev->vfl_dir == VFL_DIR_RX)
                        return v4l2_src_change_event_subscribe(fh, sub);
                break;
        default:
                return v4l2_ctrl_subscribe_event(fh, sub);
        }
        return -EINVAL;
}





























































































































































































































































































































































































   75 
   75 





























































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * kernfs.h - pseudo filesystem decoupled from vfs locking
 */

#ifndef __LINUX_KERNFS_H
#define __LINUX_KERNFS_H

#include <linux/err.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/idr.h>
#include <linux/lockdep.h>
#include <linux/rbtree.h>
#include <linux/atomic.h>
#include <linux/bug.h>
#include <linux/types.h>
#include <linux/uidgid.h>
#include <linux/wait.h>
#include <linux/rwsem.h>
#include <linux/cache.h>

struct file;
struct dentry;
struct iattr;
struct seq_file;
struct vm_area_struct;
struct vm_operations_struct;
struct super_block;
struct file_system_type;
struct poll_table_struct;
struct fs_context;

struct kernfs_fs_context;
struct kernfs_open_node;
struct kernfs_iattrs;

/*
 * NR_KERNFS_LOCK_BITS determines size (NR_KERNFS_LOCKS) of hash
 * table of locks.
 * Having a small hash table would impact scalability, since
 * more and more kernfs_node objects will end up using same lock
 * and having a very large hash table would waste memory.
 *
 * At the moment size of hash table of locks is being set based on
 * the number of CPUs as follows:
 *
 * NR_CPU      NR_KERNFS_LOCK_BITS      NR_KERNFS_LOCKS
 *   1                  1                       2
 *  2-3                 2                       4
 *  4-7                 4                       16
 *  8-15                6                       64
 *  16-31               8                       256
 *  32 and more         10                      1024
 *
 * The above relation between NR_CPU and number of locks is based
 * on some internal experimentation which involved booting qemu
 * with different values of smp, performing some sysfs operations
 * on all CPUs and observing how increase in number of locks impacts
 * completion time of these sysfs operations on each CPU.
 */
#ifdef CONFIG_SMP
#define NR_KERNFS_LOCK_BITS (2 * (ilog2(NR_CPUS < 32 ? NR_CPUS : 32)))
#else
#define NR_KERNFS_LOCK_BITS     1
#endif

#define NR_KERNFS_LOCKS     (1 << NR_KERNFS_LOCK_BITS)

/*
 * There's one kernfs_open_file for each open file and one kernfs_open_node
 * for each kernfs_node with one or more open files.
 *
 * filp->private_data points to seq_file whose ->private points to
 * kernfs_open_file.
 *
 * kernfs_open_files are chained at kernfs_open_node->files, which is
 * protected by kernfs_global_locks.open_file_mutex[i].
 *
 * To reduce possible contention in sysfs access, arising due to single
 * locks, use an array of locks (e.g. open_file_mutex) and use kernfs_node
 * object address as hash keys to get the index of these locks.
 *
 * Hashed mutexes are safe to use here because operations using these don't
 * rely on global exclusion.
 *
 * In future we intend to replace other global locks with hashed ones as well.
 * kernfs_global_locks acts as a holder for all such hash tables.
 */
struct kernfs_global_locks {
        struct mutex open_file_mutex[NR_KERNFS_LOCKS];
};

enum kernfs_node_type {
        KERNFS_DIR                = 0x0001,
        KERNFS_FILE                = 0x0002,
        KERNFS_LINK                = 0x0004,
};

#define KERNFS_TYPE_MASK                0x000f
#define KERNFS_FLAG_MASK                ~KERNFS_TYPE_MASK
#define KERNFS_MAX_USER_XATTRS                128
#define KERNFS_USER_XATTR_SIZE_LIMIT        (128 << 10)

enum kernfs_node_flag {
        KERNFS_ACTIVATED        = 0x0010,
        KERNFS_NS                = 0x0020,
        KERNFS_HAS_SEQ_SHOW        = 0x0040,
        KERNFS_HAS_MMAP                = 0x0080,
        KERNFS_LOCKDEP                = 0x0100,
        KERNFS_HIDDEN                = 0x0200,
        KERNFS_SUICIDAL                = 0x0400,
        KERNFS_SUICIDED                = 0x0800,
        KERNFS_EMPTY_DIR        = 0x1000,
        KERNFS_HAS_RELEASE        = 0x2000,
        KERNFS_REMOVING                = 0x4000,
};

/* @flags for kernfs_create_root() */
enum kernfs_root_flag {
        /*
         * kernfs_nodes are created in the deactivated state and invisible.
         * They require explicit kernfs_activate() to become visible.  This
         * can be used to make related nodes become visible atomically
         * after all nodes are created successfully.
         */
        KERNFS_ROOT_CREATE_DEACTIVATED                = 0x0001,

        /*
         * For regular files, if the opener has CAP_DAC_OVERRIDE, open(2)
         * succeeds regardless of the RW permissions.  sysfs had an extra
         * layer of enforcement where open(2) fails with -EACCES regardless
         * of CAP_DAC_OVERRIDE if the permission doesn't have the
         * respective read or write access at all (none of S_IRUGO or
         * S_IWUGO) or the respective operation isn't implemented.  The
         * following flag enables that behavior.
         */
        KERNFS_ROOT_EXTRA_OPEN_PERM_CHECK        = 0x0002,

        /*
         * The filesystem supports exportfs operation, so userspace can use
         * fhandle to access nodes of the fs.
         */
        KERNFS_ROOT_SUPPORT_EXPORTOP                = 0x0004,

        /*
         * Support user xattrs to be written to nodes rooted at this root.
         */
        KERNFS_ROOT_SUPPORT_USER_XATTR                = 0x0008,
};

/* type-specific structures for kernfs_node union members */
struct kernfs_elem_dir {
        unsigned long                subdirs;
        /* children rbtree starts here and goes through kn->rb */
        struct rb_root                children;

        /*
         * The kernfs hierarchy this directory belongs to.  This fits
         * better directly in kernfs_node but is here to save space.
         */
        struct kernfs_root        *root;
        /*
         * Monotonic revision counter, used to identify if a directory
         * node has changed during negative dentry revalidation.
         */
        unsigned long                rev;
};

struct kernfs_elem_symlink {
        struct kernfs_node        *target_kn;
};

struct kernfs_elem_attr {
        const struct kernfs_ops        *ops;
        struct kernfs_open_node __rcu        *open;
        loff_t                        size;
        struct kernfs_node        *notify_next;        /* for kernfs_notify() */
};

/*
 * kernfs_node - the building block of kernfs hierarchy.  Each and every
 * kernfs node is represented by single kernfs_node.  Most fields are
 * private to kernfs and shouldn't be accessed directly by kernfs users.
 *
 * As long as count reference is held, the kernfs_node itself is
 * accessible.  Dereferencing elem or any other outer entity requires
 * active reference.
 */
struct kernfs_node {
        atomic_t                count;
        atomic_t                active;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
        struct lockdep_map        dep_map;
#endif
        /*
         * Use kernfs_get_parent() and kernfs_name/path() instead of
         * accessing the following two fields directly.  If the node is
         * never moved to a different parent, it is safe to access the
         * parent directly.
         */
        struct kernfs_node        *parent;
        const char                *name;

        struct rb_node                rb;

        const void                *ns;        /* namespace tag */
        unsigned int                hash;        /* ns + name hash */
        unsigned short                flags;
        umode_t                        mode;

        union {
                struct kernfs_elem_dir                dir;
                struct kernfs_elem_symlink        symlink;
                struct kernfs_elem_attr                attr;
        };

        /*
         * 64bit unique ID.  On 64bit ino setups, id is the ino.  On 32bit,
         * the low 32bits are ino and upper generation.
         */
        u64                        id;

        void                        *priv;
        struct kernfs_iattrs        *iattr;

        struct rcu_head                rcu;
};

/*
 * kernfs_syscall_ops may be specified on kernfs_create_root() to support
 * syscalls.  These optional callbacks are invoked on the matching syscalls
 * and can perform any kernfs operations which don't necessarily have to be
 * the exact operation requested.  An active reference is held for each
 * kernfs_node parameter.
 */
struct kernfs_syscall_ops {
        int (*show_options)(struct seq_file *sf, struct kernfs_root *root);

        int (*mkdir)(struct kernfs_node *parent, const char *name,
                     umode_t mode);
        int (*rmdir)(struct kernfs_node *kn);
        int (*rename)(struct kernfs_node *kn, struct kernfs_node *new_parent,
                      const char *new_name);
        int (*show_path)(struct seq_file *sf, struct kernfs_node *kn,
                         struct kernfs_root *root);
};

struct kernfs_node *kernfs_root_to_node(struct kernfs_root *root);

struct kernfs_open_file {
        /* published fields */
        struct kernfs_node        *kn;
        struct file                *file;
        struct seq_file                *seq_file;
        void                        *priv;

        /* private fields, do not use outside kernfs proper */
        struct mutex                mutex;
        struct mutex                prealloc_mutex;
        int                        event;
        struct list_head        list;
        char                        *prealloc_buf;

        size_t                        atomic_write_len;
        bool                        mmapped:1;
        bool                        released:1;
        const struct vm_operations_struct *vm_ops;
};

struct kernfs_ops {
        /*
         * Optional open/release methods.  Both are called with
         * @of->seq_file populated.
         */
        int (*open)(struct kernfs_open_file *of);
        void (*release)(struct kernfs_open_file *of);

        /*
         * Read is handled by either seq_file or raw_read().
         *
         * If seq_show() is present, seq_file path is active.  Other seq
         * operations are optional and if not implemented, the behavior is
         * equivalent to single_open().  @sf->private points to the
         * associated kernfs_open_file.
         *
         * read() is bounced through kernel buffer and a read larger than
         * PAGE_SIZE results in partial operation of PAGE_SIZE.
         */
        int (*seq_show)(struct seq_file *sf, void *v);

        void *(*seq_start)(struct seq_file *sf, loff_t *ppos);
        void *(*seq_next)(struct seq_file *sf, void *v, loff_t *ppos);
        void (*seq_stop)(struct seq_file *sf, void *v);

        ssize_t (*read)(struct kernfs_open_file *of, char *buf, size_t bytes,
                        loff_t off);

        /*
         * write() is bounced through kernel buffer.  If atomic_write_len
         * is not set, a write larger than PAGE_SIZE results in partial
         * operations of PAGE_SIZE chunks.  If atomic_write_len is set,
         * writes upto the specified size are executed atomically but
         * larger ones are rejected with -E2BIG.
         */
        size_t atomic_write_len;
        /*
         * "prealloc" causes a buffer to be allocated at open for
         * all read/write requests.  As ->seq_show uses seq_read()
         * which does its own allocation, it is incompatible with
         * ->prealloc.  Provide ->read and ->write with ->prealloc.
         */
        bool prealloc;
        ssize_t (*write)(struct kernfs_open_file *of, char *buf, size_t bytes,
                         loff_t off);

        __poll_t (*poll)(struct kernfs_open_file *of,
                         struct poll_table_struct *pt);

        int (*mmap)(struct kernfs_open_file *of, struct vm_area_struct *vma);
        loff_t (*llseek)(struct kernfs_open_file *of, loff_t offset, int whence);
};

/*
 * The kernfs superblock creation/mount parameter context.
 */
struct kernfs_fs_context {
        struct kernfs_root        *root;                /* Root of the hierarchy being mounted */
        void                        *ns_tag;        /* Namespace tag of the mount (or NULL) */
        unsigned long                magic;                /* File system specific magic number */

        /* The following are set/used by kernfs_mount() */
        bool                        new_sb_created;        /* Set to T if we allocated a new sb */
};

#ifdef CONFIG_KERNFS

static inline enum kernfs_node_type kernfs_type(struct kernfs_node *kn)
{
        return kn->flags & KERNFS_TYPE_MASK;
}

static inline ino_t kernfs_id_ino(u64 id)
{
        /* id is ino if ino_t is 64bit; otherwise, low 32bits */
        if (sizeof(ino_t) >= sizeof(u64))
                return id;
        else
                return (u32)id;
}

static inline u32 kernfs_id_gen(u64 id)
{
        /* gen is fixed at 1 if ino_t is 64bit; otherwise, high 32bits */
        if (sizeof(ino_t) >= sizeof(u64))
                return 1;
        else
                return id >> 32;
}

static inline ino_t kernfs_ino(struct kernfs_node *kn)
{
        return kernfs_id_ino(kn->id);
}

static inline ino_t kernfs_gen(struct kernfs_node *kn)
{
        return kernfs_id_gen(kn->id);
}

/**
 * kernfs_enable_ns - enable namespace under a directory
 * @kn: directory of interest, should be empty
 *
 * This is to be called right after @kn is created to enable namespace
 * under it.  All children of @kn must have non-NULL namespace tags and
 * only the ones which match the super_block's tag will be visible.
 */
static inline void kernfs_enable_ns(struct kernfs_node *kn)
{
        WARN_ON_ONCE(kernfs_type(kn) != KERNFS_DIR);
        WARN_ON_ONCE(!RB_EMPTY_ROOT(&kn->dir.children));
        kn->flags |= KERNFS_NS;
}

/**
 * kernfs_ns_enabled - test whether namespace is enabled
 * @kn: the node to test
 *
 * Test whether namespace filtering is enabled for the children of @ns.
 */
static inline bool kernfs_ns_enabled(struct kernfs_node *kn)
{
        return kn->flags & KERNFS_NS;
}

int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen);
int kernfs_path_from_node(struct kernfs_node *root_kn, struct kernfs_node *kn,
                          char *buf, size_t buflen);
void pr_cont_kernfs_name(struct kernfs_node *kn);
void pr_cont_kernfs_path(struct kernfs_node *kn);
struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn);
struct kernfs_node *kernfs_find_and_get_ns(struct kernfs_node *parent,
                                           const char *name, const void *ns);
struct kernfs_node *kernfs_walk_and_get_ns(struct kernfs_node *parent,
                                           const char *path, const void *ns);
void kernfs_get(struct kernfs_node *kn);
void kernfs_put(struct kernfs_node *kn);

struct kernfs_node *kernfs_node_from_dentry(struct dentry *dentry);
struct kernfs_root *kernfs_root_from_sb(struct super_block *sb);
struct inode *kernfs_get_inode(struct super_block *sb, struct kernfs_node *kn);

struct dentry *kernfs_node_dentry(struct kernfs_node *kn,
                                  struct super_block *sb);
struct kernfs_root *kernfs_create_root(struct kernfs_syscall_ops *scops,
                                       unsigned int flags, void *priv);
void kernfs_destroy_root(struct kernfs_root *root);

struct kernfs_node *kernfs_create_dir_ns(struct kernfs_node *parent,
                                         const char *name, umode_t mode,
                                         kuid_t uid, kgid_t gid,
                                         void *priv, const void *ns);
struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent,
                                            const char *name);
struct kernfs_node *__kernfs_create_file(struct kernfs_node *parent,
                                         const char *name, umode_t mode,
                                         kuid_t uid, kgid_t gid,
                                         loff_t size,
                                         const struct kernfs_ops *ops,
                                         void *priv, const void *ns,
                                         struct lock_class_key *key);
struct kernfs_node *kernfs_create_link(struct kernfs_node *parent,
                                       const char *name,
                                       struct kernfs_node *target);
void kernfs_activate(struct kernfs_node *kn);
void kernfs_show(struct kernfs_node *kn, bool show);
void kernfs_remove(struct kernfs_node *kn);
void kernfs_break_active_protection(struct kernfs_node *kn);
void kernfs_unbreak_active_protection(struct kernfs_node *kn);
bool kernfs_remove_self(struct kernfs_node *kn);
int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name,
                             const void *ns);
int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent,
                     const char *new_name, const void *new_ns);
int kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr);
__poll_t kernfs_generic_poll(struct kernfs_open_file *of,
                             struct poll_table_struct *pt);
void kernfs_notify(struct kernfs_node *kn);

int kernfs_xattr_get(struct kernfs_node *kn, const char *name,
                     void *value, size_t size);
int kernfs_xattr_set(struct kernfs_node *kn, const char *name,
                     const void *value, size_t size, int flags);

const void *kernfs_super_ns(struct super_block *sb);
int kernfs_get_tree(struct fs_context *fc);
void kernfs_free_fs_context(struct fs_context *fc);
void kernfs_kill_sb(struct super_block *sb);

void kernfs_init(void);

struct kernfs_node *kernfs_find_and_get_node_by_id(struct kernfs_root *root,
                                                   u64 id);
#else        /* CONFIG_KERNFS */

static inline enum kernfs_node_type kernfs_type(struct kernfs_node *kn)
{ return 0; }        /* whatever */

static inline void kernfs_enable_ns(struct kernfs_node *kn) { }

static inline bool kernfs_ns_enabled(struct kernfs_node *kn)
{ return false; }

static inline int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen)
{ return -ENOSYS; }

static inline int kernfs_path_from_node(struct kernfs_node *root_kn,
                                        struct kernfs_node *kn,
                                        char *buf, size_t buflen)
{ return -ENOSYS; }

static inline void pr_cont_kernfs_name(struct kernfs_node *kn) { }
static inline void pr_cont_kernfs_path(struct kernfs_node *kn) { }

static inline struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn)
{ return NULL; }

static inline struct kernfs_node *
kernfs_find_and_get_ns(struct kernfs_node *parent, const char *name,
                       const void *ns)
{ return NULL; }
static inline struct kernfs_node *
kernfs_walk_and_get_ns(struct kernfs_node *parent, const char *path,
                       const void *ns)
{ return NULL; }

static inline void kernfs_get(struct kernfs_node *kn) { }
static inline void kernfs_put(struct kernfs_node *kn) { }

static inline struct kernfs_node *kernfs_node_from_dentry(struct dentry *dentry)
{ return NULL; }

static inline struct kernfs_root *kernfs_root_from_sb(struct super_block *sb)
{ return NULL; }

static inline struct inode *
kernfs_get_inode(struct super_block *sb, struct kernfs_node *kn)
{ return NULL; }

static inline struct kernfs_root *
kernfs_create_root(struct kernfs_syscall_ops *scops, unsigned int flags,
                   void *priv)
{ return ERR_PTR(-ENOSYS); }

static inline void kernfs_destroy_root(struct kernfs_root *root) { }

static inline struct kernfs_node *
kernfs_create_dir_ns(struct kernfs_node *parent, const char *name,
                     umode_t mode, kuid_t uid, kgid_t gid,
                     void *priv, const void *ns)
{ return ERR_PTR(-ENOSYS); }

static inline struct kernfs_node *
__kernfs_create_file(struct kernfs_node *parent, const char *name,
                     umode_t mode, kuid_t uid, kgid_t gid,
                     loff_t size, const struct kernfs_ops *ops,
                     void *priv, const void *ns, struct lock_class_key *key)
{ return ERR_PTR(-ENOSYS); }

static inline struct kernfs_node *
kernfs_create_link(struct kernfs_node *parent, const char *name,
                   struct kernfs_node *target)
{ return ERR_PTR(-ENOSYS); }

static inline void kernfs_activate(struct kernfs_node *kn) { }

static inline void kernfs_remove(struct kernfs_node *kn) { }

static inline bool kernfs_remove_self(struct kernfs_node *kn)
{ return false; }

static inline int kernfs_remove_by_name_ns(struct kernfs_node *kn,
                                           const char *name, const void *ns)
{ return -ENOSYS; }

static inline int kernfs_rename_ns(struct kernfs_node *kn,
                                   struct kernfs_node *new_parent,
                                   const char *new_name, const void *new_ns)
{ return -ENOSYS; }

static inline int kernfs_setattr(struct kernfs_node *kn,
                                 const struct iattr *iattr)
{ return -ENOSYS; }

static inline __poll_t kernfs_generic_poll(struct kernfs_open_file *of,
                                           struct poll_table_struct *pt)
{ return -ENOSYS; }

static inline void kernfs_notify(struct kernfs_node *kn) { }

static inline int kernfs_xattr_get(struct kernfs_node *kn, const char *name,
                                   void *value, size_t size)
{ return -ENOSYS; }

static inline int kernfs_xattr_set(struct kernfs_node *kn, const char *name,
                                   const void *value, size_t size, int flags)
{ return -ENOSYS; }

static inline const void *kernfs_super_ns(struct super_block *sb)
{ return NULL; }

static inline int kernfs_get_tree(struct fs_context *fc)
{ return -ENOSYS; }

static inline void kernfs_free_fs_context(struct fs_context *fc) { }

static inline void kernfs_kill_sb(struct super_block *sb) { }

static inline void kernfs_init(void) { }

#endif        /* CONFIG_KERNFS */

/**
 * kernfs_path - build full path of a given node
 * @kn: kernfs_node of interest
 * @buf: buffer to copy @kn's name into
 * @buflen: size of @buf
 *
 * If @kn is NULL result will be "(null)".
 *
 * Returns the length of the full path.  If the full length is equal to or
 * greater than @buflen, @buf contains the truncated path with the trailing
 * '\0'.  On error, -errno is returned.
 */
static inline int kernfs_path(struct kernfs_node *kn, char *buf, size_t buflen)
{
        return kernfs_path_from_node(kn, NULL, buf, buflen);
}

static inline struct kernfs_node *
kernfs_find_and_get(struct kernfs_node *kn, const char *name)
{
        return kernfs_find_and_get_ns(kn, name, NULL);
}

static inline struct kernfs_node *
kernfs_walk_and_get(struct kernfs_node *kn, const char *path)
{
        return kernfs_walk_and_get_ns(kn, path, NULL);
}

static inline struct kernfs_node *
kernfs_create_dir(struct kernfs_node *parent, const char *name, umode_t mode,
                  void *priv)
{
        return kernfs_create_dir_ns(parent, name, mode,
                                    GLOBAL_ROOT_UID, GLOBAL_ROOT_GID,
                                    priv, NULL);
}

static inline int kernfs_remove_by_name(struct kernfs_node *parent,
                                        const char *name)
{
        return kernfs_remove_by_name_ns(parent, name, NULL);
}

static inline int kernfs_rename(struct kernfs_node *kn,
                                struct kernfs_node *new_parent,
                                const char *new_name)
{
        return kernfs_rename_ns(kn, new_parent, new_name, NULL);
}

#endif        /* __LINUX_KERNFS_H */

























    8 


    2 






    1 











    4 








    1 
    3 




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
// SPDX-License-Identifier: GPL-2.0
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/uaccess.h>
#include <linux/io_uring.h>
#include <linux/eventpoll.h>

#include <uapi/linux/io_uring.h>

#include "io_uring.h"
#include "epoll.h"

#if defined(CONFIG_EPOLL)
struct io_epoll {
        struct file                        *file;
        int                                epfd;
        int                                op;
        int                                fd;
        struct epoll_event                event;
};

int io_epoll_ctl_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
        struct io_epoll *epoll = io_kiocb_to_cmd(req, struct io_epoll);

        if (sqe->buf_index || sqe->splice_fd_in)
                return -EINVAL;

        epoll->epfd = READ_ONCE(sqe->fd);
        epoll->op = READ_ONCE(sqe->len);
        epoll->fd = READ_ONCE(sqe->off);

        if (ep_op_has_event(epoll->op)) {
                struct epoll_event __user *ev;

                ev = u64_to_user_ptr(READ_ONCE(sqe->addr));
                if (copy_from_user(&epoll->event, ev, sizeof(*ev)))
                        return -EFAULT;
        }

        return 0;
}

int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags)
{
        struct io_epoll *ie = io_kiocb_to_cmd(req, struct io_epoll);
        int ret;
        bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;

        ret = do_epoll_ctl(ie->epfd, ie->op, ie->fd, &ie->event, force_nonblock);
        if (force_nonblock && ret == -EAGAIN)
                return -EAGAIN;

        if (ret < 0)
                req_set_fail(req);
        io_req_set_res(req, ret, 0);
        return IOU_OK;
}
#endif

































































































































































































































   79 










   70 






  236 






















  236 























    1 

































  236 











































 1157 















   41 
  222 












   15 
   11 

















































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _NDISC_H
#define _NDISC_H

#include <net/ipv6_stubs.h>

/*
 *        ICMP codes for neighbour discovery messages
 */

#define NDISC_ROUTER_SOLICITATION        133
#define NDISC_ROUTER_ADVERTISEMENT        134
#define NDISC_NEIGHBOUR_SOLICITATION        135
#define NDISC_NEIGHBOUR_ADVERTISEMENT        136
#define NDISC_REDIRECT                        137

/*
 * Router type: cross-layer information from link-layer to
 * IPv6 layer reported by certain link types (e.g., RFC4214).
 */
#define NDISC_NODETYPE_UNSPEC                0        /* unspecified (default) */
#define NDISC_NODETYPE_HOST                1        /* host or unauthorized router */
#define NDISC_NODETYPE_NODEFAULT        2        /* non-default router */
#define NDISC_NODETYPE_DEFAULT                3        /* default router */

/*
 *        ndisc options
 */

enum {
        __ND_OPT_PREFIX_INFO_END = 0,
        ND_OPT_SOURCE_LL_ADDR = 1,        /* RFC2461 */
        ND_OPT_TARGET_LL_ADDR = 2,        /* RFC2461 */
        ND_OPT_PREFIX_INFO = 3,                /* RFC2461 */
        ND_OPT_REDIRECT_HDR = 4,        /* RFC2461 */
        ND_OPT_MTU = 5,                        /* RFC2461 */
        ND_OPT_NONCE = 14,              /* RFC7527 */
        __ND_OPT_ARRAY_MAX,
        ND_OPT_ROUTE_INFO = 24,                /* RFC4191 */
        ND_OPT_RDNSS = 25,                /* RFC5006 */
        ND_OPT_DNSSL = 31,                /* RFC6106 */
        ND_OPT_6CO = 34,                /* RFC6775 */
        ND_OPT_CAPTIVE_PORTAL = 37,        /* RFC7710 */
        ND_OPT_PREF64 = 38,                /* RFC8781 */
        __ND_OPT_MAX
};

#define MAX_RTR_SOLICITATION_DELAY        HZ

#define ND_REACHABLE_TIME                (30*HZ)
#define ND_RETRANS_TIMER                HZ

#include <linux/compiler.h>
#include <linux/icmpv6.h>
#include <linux/in6.h>
#include <linux/types.h>
#include <linux/if_arp.h>
#include <linux/netdevice.h>
#include <linux/hash.h>

#include <net/neighbour.h>

/* Set to 3 to get tracing... */
#define ND_DEBUG 1

#define ND_PRINTK(val, level, fmt, ...)                                \
do {                                                                \
        if (val <= ND_DEBUG)                                        \
                net_##level##_ratelimited(fmt, ##__VA_ARGS__);        \
} while (0)

struct ctl_table;
struct inet6_dev;
struct net_device;
struct net_proto_family;
struct sk_buff;
struct prefix_info;

extern struct neigh_table nd_tbl;

struct nd_msg {
        struct icmp6hdr        icmph;
        struct in6_addr        target;
        __u8                opt[];
};

struct rs_msg {
        struct icmp6hdr        icmph;
        __u8                opt[];
};

struct ra_msg {
        struct icmp6hdr                icmph;
        __be32                        reachable_time;
        __be32                        retrans_timer;
};

struct rd_msg {
        struct icmp6hdr icmph;
        struct in6_addr        target;
        struct in6_addr        dest;
        __u8                opt[];
};

struct nd_opt_hdr {
        __u8                nd_opt_type;
        __u8                nd_opt_len;
} __packed;

/* ND options */
struct ndisc_options {
        struct nd_opt_hdr *nd_opt_array[__ND_OPT_ARRAY_MAX];
#ifdef CONFIG_IPV6_ROUTE_INFO
        struct nd_opt_hdr *nd_opts_ri;
        struct nd_opt_hdr *nd_opts_ri_end;
#endif
        struct nd_opt_hdr *nd_useropts;
        struct nd_opt_hdr *nd_useropts_end;
#if IS_ENABLED(CONFIG_IEEE802154_6LOWPAN)
        struct nd_opt_hdr *nd_802154_opt_array[ND_OPT_TARGET_LL_ADDR + 1];
#endif
};

#define nd_opts_src_lladdr                nd_opt_array[ND_OPT_SOURCE_LL_ADDR]
#define nd_opts_tgt_lladdr                nd_opt_array[ND_OPT_TARGET_LL_ADDR]
#define nd_opts_pi                        nd_opt_array[ND_OPT_PREFIX_INFO]
#define nd_opts_pi_end                        nd_opt_array[__ND_OPT_PREFIX_INFO_END]
#define nd_opts_rh                        nd_opt_array[ND_OPT_REDIRECT_HDR]
#define nd_opts_mtu                        nd_opt_array[ND_OPT_MTU]
#define nd_opts_nonce                        nd_opt_array[ND_OPT_NONCE]
#define nd_802154_opts_src_lladdr        nd_802154_opt_array[ND_OPT_SOURCE_LL_ADDR]
#define nd_802154_opts_tgt_lladdr        nd_802154_opt_array[ND_OPT_TARGET_LL_ADDR]

#define NDISC_OPT_SPACE(len) (((len)+2+7)&~7)

struct ndisc_options *ndisc_parse_options(const struct net_device *dev,
                                          u8 *opt, int opt_len,
                                          struct ndisc_options *ndopts);

void __ndisc_fill_addr_option(struct sk_buff *skb, int type, const void *data,
                              int data_len, int pad);

#define NDISC_OPS_REDIRECT_DATA_SPACE        2

/*
 * This structure defines the hooks for IPv6 neighbour discovery.
 * The following hooks can be defined; unless noted otherwise, they are
 * optional and can be filled with a null pointer.
 *
 * int (*parse_options)(const struct net_device *dev,
 *                        struct nd_opt_hdr *nd_opt,
 *                        struct ndisc_options *ndopts):
 *     This function is called while parsing ndisc ops and put each position
 *     as pointer into ndopts. If this function return unequal 0, then this
 *     function took care about the ndisc option, if 0 then the IPv6 ndisc
 *     option parser will take care about that option.
 *
 * void (*update)(const struct net_device *dev, struct neighbour *n,
 *                  u32 flags, u8 icmp6_type,
 *                  const struct ndisc_options *ndopts):
 *     This function is called when IPv6 ndisc updates the neighbour cache
 *     entry. Additional options which can be updated may be previously
 *     parsed by parse_opts callback and accessible over ndopts parameter.
 *
 * int (*opt_addr_space)(const struct net_device *dev, u8 icmp6_type,
 *                         struct neighbour *neigh, u8 *ha_buf,
 *                         u8 **ha):
 *     This function is called when the necessary option space will be
 *     calculated before allocating a skb. The parameters neigh, ha_buf
 *     abd ha are available on NDISC_REDIRECT messages only.
 *
 * void (*fill_addr_option)(const struct net_device *dev,
 *                            struct sk_buff *skb, u8 icmp6_type,
 *                            const u8 *ha):
 *     This function is called when the skb will finally fill the option
 *     fields inside skb. NOTE: this callback should fill the option
 *     fields to the skb which are previously indicated by opt_space
 *     parameter. That means the decision to add such option should
 *     not lost between these two callbacks, e.g. protected by interface
 *     up state.
 *
 * void (*prefix_rcv_add_addr)(struct net *net, struct net_device *dev,
 *                               const struct prefix_info *pinfo,
 *                               struct inet6_dev *in6_dev,
 *                               struct in6_addr *addr,
 *                               int addr_type, u32 addr_flags,
 *                               bool sllao, bool tokenized,
 *                               __u32 valid_lft, u32 prefered_lft,
 *                               bool dev_addr_generated):
 *     This function is called when a RA messages is received with valid
 *     PIO option fields and an IPv6 address will be added to the interface
 *     for autoconfiguration. The parameter dev_addr_generated reports about
 *     if the address was based on dev->dev_addr or not. This can be used
 *     to add a second address if link-layer operates with two link layer
 *     addresses. E.g. 802.15.4 6LoWPAN.
 */
struct ndisc_ops {
        int        (*parse_options)(const struct net_device *dev,
                                 struct nd_opt_hdr *nd_opt,
                                 struct ndisc_options *ndopts);
        void        (*update)(const struct net_device *dev, struct neighbour *n,
                          u32 flags, u8 icmp6_type,
                          const struct ndisc_options *ndopts);
        int        (*opt_addr_space)(const struct net_device *dev, u8 icmp6_type,
                                  struct neighbour *neigh, u8 *ha_buf,
                                  u8 **ha);
        void        (*fill_addr_option)(const struct net_device *dev,
                                    struct sk_buff *skb, u8 icmp6_type,
                                    const u8 *ha);
        void        (*prefix_rcv_add_addr)(struct net *net, struct net_device *dev,
                                       const struct prefix_info *pinfo,
                                       struct inet6_dev *in6_dev,
                                       struct in6_addr *addr,
                                       int addr_type, u32 addr_flags,
                                       bool sllao, bool tokenized,
                                       __u32 valid_lft, u32 prefered_lft,
                                       bool dev_addr_generated);
};

#if IS_ENABLED(CONFIG_IPV6)
static inline int ndisc_ops_parse_options(const struct net_device *dev,
                                          struct nd_opt_hdr *nd_opt,
                                          struct ndisc_options *ndopts)
{
        if (dev->ndisc_ops && dev->ndisc_ops->parse_options)
                return dev->ndisc_ops->parse_options(dev, nd_opt, ndopts);
        else
                return 0;
}

static inline void ndisc_ops_update(const struct net_device *dev,
                                          struct neighbour *n, u32 flags,
                                          u8 icmp6_type,
                                          const struct ndisc_options *ndopts)
{
        if (dev->ndisc_ops && dev->ndisc_ops->update)
                dev->ndisc_ops->update(dev, n, flags, icmp6_type, ndopts);
}

static inline int ndisc_ops_opt_addr_space(const struct net_device *dev,
                                           u8 icmp6_type)
{
        if (dev->ndisc_ops && dev->ndisc_ops->opt_addr_space &&
            icmp6_type != NDISC_REDIRECT)
                return dev->ndisc_ops->opt_addr_space(dev, icmp6_type, NULL,
                                                      NULL, NULL);
        else
                return 0;
}

static inline int ndisc_ops_redirect_opt_addr_space(const struct net_device *dev,
                                                    struct neighbour *neigh,
                                                    u8 *ha_buf, u8 **ha)
{
        if (dev->ndisc_ops && dev->ndisc_ops->opt_addr_space)
                return dev->ndisc_ops->opt_addr_space(dev, NDISC_REDIRECT,
                                                      neigh, ha_buf, ha);
        else
                return 0;
}

static inline void ndisc_ops_fill_addr_option(const struct net_device *dev,
                                              struct sk_buff *skb,
                                              u8 icmp6_type)
{
        if (dev->ndisc_ops && dev->ndisc_ops->fill_addr_option &&
            icmp6_type != NDISC_REDIRECT)
                dev->ndisc_ops->fill_addr_option(dev, skb, icmp6_type, NULL);
}

static inline void ndisc_ops_fill_redirect_addr_option(const struct net_device *dev,
                                                       struct sk_buff *skb,
                                                       const u8 *ha)
{
        if (dev->ndisc_ops && dev->ndisc_ops->fill_addr_option)
                dev->ndisc_ops->fill_addr_option(dev, skb, NDISC_REDIRECT, ha);
}

static inline void ndisc_ops_prefix_rcv_add_addr(struct net *net,
                                                 struct net_device *dev,
                                                 const struct prefix_info *pinfo,
                                                 struct inet6_dev *in6_dev,
                                                 struct in6_addr *addr,
                                                 int addr_type, u32 addr_flags,
                                                 bool sllao, bool tokenized,
                                                 __u32 valid_lft,
                                                 u32 prefered_lft,
                                                 bool dev_addr_generated)
{
        if (dev->ndisc_ops && dev->ndisc_ops->prefix_rcv_add_addr)
                dev->ndisc_ops->prefix_rcv_add_addr(net, dev, pinfo, in6_dev,
                                                    addr, addr_type,
                                                    addr_flags, sllao,
                                                    tokenized, valid_lft,
                                                    prefered_lft,
                                                    dev_addr_generated);
}
#endif

/*
 * Return the padding between the option length and the start of the
 * link addr.  Currently only IP-over-InfiniBand needs this, although
 * if RFC 3831 IPv6-over-Fibre Channel is ever implemented it may
 * also need a pad of 2.
 */
static inline int ndisc_addr_option_pad(unsigned short type)
{
        switch (type) {
        case ARPHRD_INFINIBAND: return 2;
        default:                return 0;
        }
}

static inline int __ndisc_opt_addr_space(unsigned char addr_len, int pad)
{
        return NDISC_OPT_SPACE(addr_len + pad);
}

#if IS_ENABLED(CONFIG_IPV6)
static inline int ndisc_opt_addr_space(struct net_device *dev, u8 icmp6_type)
{
        return __ndisc_opt_addr_space(dev->addr_len,
                                      ndisc_addr_option_pad(dev->type)) +
                ndisc_ops_opt_addr_space(dev, icmp6_type);
}

static inline int ndisc_redirect_opt_addr_space(struct net_device *dev,
                                                struct neighbour *neigh,
                                                u8 *ops_data_buf,
                                                u8 **ops_data)
{
        return __ndisc_opt_addr_space(dev->addr_len,
                                      ndisc_addr_option_pad(dev->type)) +
                ndisc_ops_redirect_opt_addr_space(dev, neigh, ops_data_buf,
                                                  ops_data);
}
#endif

static inline u8 *__ndisc_opt_addr_data(struct nd_opt_hdr *p,
                                        unsigned char addr_len, int prepad)
{
        u8 *lladdr = (u8 *)(p + 1);
        int lladdrlen = p->nd_opt_len << 3;
        if (lladdrlen != __ndisc_opt_addr_space(addr_len, prepad))
                return NULL;
        return lladdr + prepad;
}

static inline u8 *ndisc_opt_addr_data(struct nd_opt_hdr *p,
                                      struct net_device *dev)
{
        return __ndisc_opt_addr_data(p, dev->addr_len,
                                     ndisc_addr_option_pad(dev->type));
}

static inline u32 ndisc_hashfn(const void *pkey, const struct net_device *dev, __u32 *hash_rnd)
{
        const u32 *p32 = pkey;

        return (((p32[0] ^ hash32_ptr(dev)) * hash_rnd[0]) +
                (p32[1] * hash_rnd[1]) +
                (p32[2] * hash_rnd[2]) +
                (p32[3] * hash_rnd[3]));
}

static inline struct neighbour *__ipv6_neigh_lookup_noref(struct net_device *dev, const void *pkey)
{
        return ___neigh_lookup_noref(&nd_tbl, neigh_key_eq128, ndisc_hashfn, pkey, dev);
}

static inline
struct neighbour *__ipv6_neigh_lookup_noref_stub(struct net_device *dev,
                                                 const void *pkey)
{
        return ___neigh_lookup_noref(ipv6_stub->nd_tbl, neigh_key_eq128,
                                     ndisc_hashfn, pkey, dev);
}

static inline struct neighbour *__ipv6_neigh_lookup(struct net_device *dev, const void *pkey)
{
        struct neighbour *n;

        rcu_read_lock();
        n = __ipv6_neigh_lookup_noref(dev, pkey);
        if (n && !refcount_inc_not_zero(&n->refcnt))
                n = NULL;
        rcu_read_unlock();

        return n;
}

static inline void __ipv6_confirm_neigh(struct net_device *dev,
                                        const void *pkey)
{
        struct neighbour *n;

        rcu_read_lock();
        n = __ipv6_neigh_lookup_noref(dev, pkey);
        neigh_confirm(n);
        rcu_read_unlock();
}

static inline void __ipv6_confirm_neigh_stub(struct net_device *dev,
                                             const void *pkey)
{
        struct neighbour *n;

        rcu_read_lock();
        n = __ipv6_neigh_lookup_noref_stub(dev, pkey);
        neigh_confirm(n);
        rcu_read_unlock();
}

/* uses ipv6_stub and is meant for use outside of IPv6 core */
static inline struct neighbour *ip_neigh_gw6(struct net_device *dev,
                                             const void *addr)
{
        struct neighbour *neigh;

        neigh = __ipv6_neigh_lookup_noref_stub(dev, addr);
        if (unlikely(!neigh))
                neigh = __neigh_create(ipv6_stub->nd_tbl, addr, dev, false);

        return neigh;
}

int ndisc_init(void);
int ndisc_late_init(void);

void ndisc_late_cleanup(void);
void ndisc_cleanup(void);

enum skb_drop_reason ndisc_rcv(struct sk_buff *skb);

struct sk_buff *ndisc_ns_create(struct net_device *dev, const struct in6_addr *solicit,
                                const struct in6_addr *saddr, u64 nonce);
void ndisc_send_ns(struct net_device *dev, const struct in6_addr *solicit,
                   const struct in6_addr *daddr, const struct in6_addr *saddr,
                   u64 nonce);

void ndisc_send_skb(struct sk_buff *skb, const struct in6_addr *daddr,
                    const struct in6_addr *saddr);

void ndisc_send_rs(struct net_device *dev,
                   const struct in6_addr *saddr, const struct in6_addr *daddr);
void ndisc_send_na(struct net_device *dev, const struct in6_addr *daddr,
                   const struct in6_addr *solicited_addr,
                   bool router, bool solicited, bool override, bool inc_opt);

void ndisc_send_redirect(struct sk_buff *skb, const struct in6_addr *target);

int ndisc_mc_map(const struct in6_addr *addr, char *buf, struct net_device *dev,
                 int dir);

void ndisc_update(const struct net_device *dev, struct neighbour *neigh,
                  const u8 *lladdr, u8 new, u32 flags, u8 icmp6_type,
                  struct ndisc_options *ndopts);

/*
 *        IGMP
 */
int igmp6_init(void);
int igmp6_late_init(void);

void igmp6_cleanup(void);
void igmp6_late_cleanup(void);

void igmp6_event_query(struct sk_buff *skb);

void igmp6_event_report(struct sk_buff *skb);


#ifdef CONFIG_SYSCTL
int ndisc_ifinfo_sysctl_change(const struct ctl_table *ctl, int write,
                               void *buffer, size_t *lenp, loff_t *ppos);
#endif

void inet6_ifinfo_notify(int event, struct inet6_dev *idev);

#endif


























































































  114 





   79 









  110 






   12 


    2 

    7 









   39 




   36 






   79 


   35 



   35 













   17 





    5 

    2 






   10 

















































    7 












    7 



































































    1 














































    6 


    1 

    1 

   10 





































































































































































































































































































































































    1 





    1 
























































    1 










    1 










    1 
    1 
    1 










































































































































































































































































































































































































































































































































































































































































































































































































































   29 







   26 



    7 









    7 






    7 


































  260 













  235 
   56 





   29 
























  242 


































  228 





   10 
    2 



   12 


   11 
    1 


   12 



    6 
    6 
    2 















    2 


    9 
    7 



    7 
    5 
    1 







   11 


   11 

   10 
    1 


   11 

   11 











   11 











    3 
    8 








    5 























    1 



















    1 




















































































































   86 






   37 
























   30 


   40 





















   87 




   87 












   64 




   64 
































    3 

















    3 













    1 

    3 























    3 











































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * USB Network driver infrastructure
 * Copyright (C) 2000-2005 by David Brownell
 * Copyright (C) 2003-2005 David Hollis <dhollis@davehollis.com>
 */

/*
 * This is a generic "USB networking" framework that works with several
 * kinds of full and high speed networking devices:  host-to-host cables,
 * smart usb peripherals, and actual Ethernet adapters.
 *
 * These devices usually differ in terms of control protocols (if they
 * even have one!) and sometimes they define new framing to wrap or batch
 * Ethernet packets.  Otherwise, they talk to USB pretty much the same,
 * so interface (un)binding, endpoint I/O queues, fault handling, and other
 * issues can usefully be addressed by this framework.
 */

#include <linux/module.h>
#include <linux/init.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/ctype.h>
#include <linux/ethtool.h>
#include <linux/workqueue.h>
#include <linux/mii.h>
#include <linux/usb.h>
#include <linux/usb/usbnet.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/pm_runtime.h>

/*-------------------------------------------------------------------------*/

/*
 * Nineteen USB 1.1 max size bulk transactions per frame (ms), max.
 * Several dozen bytes of IPv4 data can fit in two such transactions.
 * One maximum size Ethernet packet takes twenty four of them.
 * For high speed, each frame comfortably fits almost 36 max size
 * Ethernet packets (so queues should be bigger).
 *
 * The goal is to let the USB host controller be busy for 5msec or
 * more before an irq is required, under load.  Jumbograms change
 * the equation.
 */
#define        MAX_QUEUE_MEMORY        (60 * 1518)
#define        RX_QLEN(dev)                ((dev)->rx_qlen)
#define        TX_QLEN(dev)                ((dev)->tx_qlen)

// reawaken network queue this soon after stopping; else watchdog barks
#define TX_TIMEOUT_JIFFIES        (5*HZ)

/* throttle rx/tx briefly after some faults, so hub_wq might disconnect()
 * us (it polls at HZ/4 usually) before we report too many false errors.
 */
#define THROTTLE_JIFFIES        (HZ/8)

// between wakeups
#define UNLINK_TIMEOUT_MS        3

/*-------------------------------------------------------------------------*/

/* use ethtool to change the level for any given device */
static int msg_level = -1;
module_param (msg_level, int, 0);
MODULE_PARM_DESC (msg_level, "Override default message level");

/*-------------------------------------------------------------------------*/

static const char * const usbnet_event_names[] = {
        [EVENT_TX_HALT]                   = "EVENT_TX_HALT",
        [EVENT_RX_HALT]                   = "EVENT_RX_HALT",
        [EVENT_RX_MEMORY]           = "EVENT_RX_MEMORY",
        [EVENT_STS_SPLIT]           = "EVENT_STS_SPLIT",
        [EVENT_LINK_RESET]           = "EVENT_LINK_RESET",
        [EVENT_RX_PAUSED]           = "EVENT_RX_PAUSED",
        [EVENT_DEV_ASLEEP]           = "EVENT_DEV_ASLEEP",
        [EVENT_DEV_OPEN]           = "EVENT_DEV_OPEN",
        [EVENT_DEVICE_REPORT_IDLE] = "EVENT_DEVICE_REPORT_IDLE",
        [EVENT_NO_RUNTIME_PM]           = "EVENT_NO_RUNTIME_PM",
        [EVENT_RX_KILL]                   = "EVENT_RX_KILL",
        [EVENT_LINK_CHANGE]           = "EVENT_LINK_CHANGE",
        [EVENT_SET_RX_MODE]           = "EVENT_SET_RX_MODE",
        [EVENT_NO_IP_ALIGN]           = "EVENT_NO_IP_ALIGN",
};

/* handles CDC Ethernet and many other network "bulk data" interfaces */
int usbnet_get_endpoints(struct usbnet *dev, struct usb_interface *intf)
{
        int                                tmp;
        struct usb_host_interface        *alt = NULL;
        struct usb_host_endpoint        *in = NULL, *out = NULL;
        struct usb_host_endpoint        *status = NULL;

        for (tmp = 0; tmp < intf->num_altsetting; tmp++) {
                unsigned        ep;

                in = out = status = NULL;
                alt = intf->altsetting + tmp;

                /* take the first altsetting with in-bulk + out-bulk;
                 * remember any status endpoint, just in case;
                 * ignore other endpoints and altsettings.
                 */
                for (ep = 0; ep < alt->desc.bNumEndpoints; ep++) {
                        struct usb_host_endpoint        *e;
                        int                                intr = 0;

                        e = alt->endpoint + ep;

                        /* ignore endpoints which cannot transfer data */
                        if (!usb_endpoint_maxp(&e->desc))
                                continue;

                        switch (e->desc.bmAttributes) {
                        case USB_ENDPOINT_XFER_INT:
                                if (!usb_endpoint_dir_in(&e->desc))
                                        continue;
                                intr = 1;
                                fallthrough;
                        case USB_ENDPOINT_XFER_BULK:
                                break;
                        default:
                                continue;
                        }
                        if (usb_endpoint_dir_in(&e->desc)) {
                                if (!intr && !in)
                                        in = e;
                                else if (intr && !status)
                                        status = e;
                        } else {
                                if (!out)
                                        out = e;
                        }
                }
                if (in && out)
                        break;
        }
        if (!alt || !in || !out)
                return -EINVAL;

        if (alt->desc.bAlternateSetting != 0 ||
            !(dev->driver_info->flags & FLAG_NO_SETINT)) {
                tmp = usb_set_interface (dev->udev, alt->desc.bInterfaceNumber,
                                alt->desc.bAlternateSetting);
                if (tmp < 0)
                        return tmp;
        }

        dev->in = usb_rcvbulkpipe (dev->udev,
                        in->desc.bEndpointAddress & USB_ENDPOINT_NUMBER_MASK);
        dev->out = usb_sndbulkpipe (dev->udev,
                        out->desc.bEndpointAddress & USB_ENDPOINT_NUMBER_MASK);
        dev->status = status;
        return 0;
}
EXPORT_SYMBOL_GPL(usbnet_get_endpoints);

int usbnet_get_ethernet_addr(struct usbnet *dev, int iMACAddress)
{
        u8                addr[ETH_ALEN];
        int                 tmp = -1, ret;
        unsigned char        buf [13];

        ret = usb_string(dev->udev, iMACAddress, buf, sizeof buf);
        if (ret == 12)
                tmp = hex2bin(addr, buf, 6);
        if (tmp < 0) {
                dev_dbg(&dev->udev->dev,
                        "bad MAC string %d fetch, %d\n", iMACAddress, tmp);
                if (ret >= 0)
                        ret = -EINVAL;
                return ret;
        }
        eth_hw_addr_set(dev->net, addr);
        return 0;
}
EXPORT_SYMBOL_GPL(usbnet_get_ethernet_addr);

static void intr_complete (struct urb *urb)
{
        struct usbnet        *dev = urb->context;
        int                status = urb->status;

        switch (status) {
        /* success */
        case 0:
                dev->driver_info->status(dev, urb);
                break;

        /* software-driven interface shutdown */
        case -ENOENT:                /* urb killed */
        case -ESHUTDOWN:        /* hardware gone */
                netif_dbg(dev, ifdown, dev->net,
                          "intr shutdown, code %d\n", status);
                return;

        /* NOTE:  not throttling like RX/TX, since this endpoint
         * already polls infrequently
         */
        default:
                netdev_dbg(dev->net, "intr status %d\n", status);
                break;
        }

        status = usb_submit_urb (urb, GFP_ATOMIC);
        if (status != 0)
                netif_err(dev, timer, dev->net,
                          "intr resubmit --> %d\n", status);
}

static int init_status (struct usbnet *dev, struct usb_interface *intf)
{
        char                *buf = NULL;
        unsigned        pipe = 0;
        unsigned        maxp;
        unsigned        period;

        if (!dev->driver_info->status)
                return 0;

        pipe = usb_rcvintpipe (dev->udev,
                        dev->status->desc.bEndpointAddress
                                & USB_ENDPOINT_NUMBER_MASK);
        maxp = usb_maxpacket(dev->udev, pipe);

        /* avoid 1 msec chatter:  min 8 msec poll rate */
        period = max ((int) dev->status->desc.bInterval,
                (dev->udev->speed == USB_SPEED_HIGH) ? 7 : 3);

        buf = kmalloc (maxp, GFP_KERNEL);
        if (buf) {
                dev->interrupt = usb_alloc_urb (0, GFP_KERNEL);
                if (!dev->interrupt) {
                        kfree (buf);
                        return -ENOMEM;
                } else {
                        usb_fill_int_urb(dev->interrupt, dev->udev, pipe,
                                buf, maxp, intr_complete, dev, period);
                        dev->interrupt->transfer_flags |= URB_FREE_BUFFER;
                        dev_dbg(&intf->dev,
                                "status ep%din, %d bytes period %d\n",
                                usb_pipeendpoint(pipe), maxp, period);
                }
        }
        return 0;
}

/* Submit the interrupt URB if not previously submitted, increasing refcount */
int usbnet_status_start(struct usbnet *dev, gfp_t mem_flags)
{
        int ret = 0;

        WARN_ON_ONCE(dev->interrupt == NULL);
        if (dev->interrupt) {
                mutex_lock(&dev->interrupt_mutex);

                if (++dev->interrupt_count == 1)
                        ret = usb_submit_urb(dev->interrupt, mem_flags);

                dev_dbg(&dev->udev->dev, "incremented interrupt URB count to %d\n",
                        dev->interrupt_count);
                mutex_unlock(&dev->interrupt_mutex);
        }
        return ret;
}
EXPORT_SYMBOL_GPL(usbnet_status_start);

/* For resume; submit interrupt URB if previously submitted */
static int __usbnet_status_start_force(struct usbnet *dev, gfp_t mem_flags)
{
        int ret = 0;

        mutex_lock(&dev->interrupt_mutex);
        if (dev->interrupt_count) {
                ret = usb_submit_urb(dev->interrupt, mem_flags);
                dev_dbg(&dev->udev->dev,
                        "submitted interrupt URB for resume\n");
        }
        mutex_unlock(&dev->interrupt_mutex);
        return ret;
}

/* Kill the interrupt URB if all submitters want it killed */
void usbnet_status_stop(struct usbnet *dev)
{
        if (dev->interrupt) {
                mutex_lock(&dev->interrupt_mutex);
                WARN_ON(dev->interrupt_count == 0);

                if (dev->interrupt_count && --dev->interrupt_count == 0)
                        usb_kill_urb(dev->interrupt);

                dev_dbg(&dev->udev->dev,
                        "decremented interrupt URB count to %d\n",
                        dev->interrupt_count);
                mutex_unlock(&dev->interrupt_mutex);
        }
}
EXPORT_SYMBOL_GPL(usbnet_status_stop);

/* For suspend; always kill interrupt URB */
static void __usbnet_status_stop_force(struct usbnet *dev)
{
        if (dev->interrupt) {
                mutex_lock(&dev->interrupt_mutex);
                usb_kill_urb(dev->interrupt);
                dev_dbg(&dev->udev->dev, "killed interrupt URB for suspend\n");
                mutex_unlock(&dev->interrupt_mutex);
        }
}

/* Passes this packet up the stack, updating its accounting.
 * Some link protocols batch packets, so their rx_fixup paths
 * can return clones as well as just modify the original skb.
 */
void usbnet_skb_return (struct usbnet *dev, struct sk_buff *skb)
{
        struct pcpu_sw_netstats *stats64 = this_cpu_ptr(dev->net->tstats);
        unsigned long flags;
        int        status;

        if (test_bit(EVENT_RX_PAUSED, &dev->flags)) {
                skb_queue_tail(&dev->rxq_pause, skb);
                return;
        }

        /* only update if unset to allow minidriver rx_fixup override */
        if (skb->protocol == 0)
                skb->protocol = eth_type_trans (skb, dev->net);

        flags = u64_stats_update_begin_irqsave(&stats64->syncp);
        u64_stats_inc(&stats64->rx_packets);
        u64_stats_add(&stats64->rx_bytes, skb->len);
        u64_stats_update_end_irqrestore(&stats64->syncp, flags);

        netif_dbg(dev, rx_status, dev->net, "< rx, len %zu, type 0x%x\n",
                  skb->len + sizeof (struct ethhdr), skb->protocol);
        memset (skb->cb, 0, sizeof (struct skb_data));

        if (skb_defer_rx_timestamp(skb))
                return;

        status = netif_rx (skb);
        if (status != NET_RX_SUCCESS)
                netif_dbg(dev, rx_err, dev->net,
                          "netif_rx status %d\n", status);
}
EXPORT_SYMBOL_GPL(usbnet_skb_return);

/* must be called if hard_mtu or rx_urb_size changed */
void usbnet_update_max_qlen(struct usbnet *dev)
{
        enum usb_device_speed speed = dev->udev->speed;

        if (!dev->rx_urb_size || !dev->hard_mtu)
                goto insanity;
        switch (speed) {
        case USB_SPEED_HIGH:
                dev->rx_qlen = MAX_QUEUE_MEMORY / dev->rx_urb_size;
                dev->tx_qlen = MAX_QUEUE_MEMORY / dev->hard_mtu;
                break;
        case USB_SPEED_SUPER:
        case USB_SPEED_SUPER_PLUS:
                /*
                 * Not take default 5ms qlen for super speed HC to
                 * save memory, and iperf tests show 2.5ms qlen can
                 * work well
                 */
                dev->rx_qlen = 5 * MAX_QUEUE_MEMORY / dev->rx_urb_size;
                dev->tx_qlen = 5 * MAX_QUEUE_MEMORY / dev->hard_mtu;
                break;
        default:
insanity:
                dev->rx_qlen = dev->tx_qlen = 4;
        }
}
EXPORT_SYMBOL_GPL(usbnet_update_max_qlen);


/*-------------------------------------------------------------------------
 *
 * Network Device Driver (peer link to "Host Device", from USB host)
 *
 *-------------------------------------------------------------------------*/

int usbnet_change_mtu (struct net_device *net, int new_mtu)
{
        struct usbnet        *dev = netdev_priv(net);
        int                ll_mtu = new_mtu + net->hard_header_len;
        int                old_hard_mtu = dev->hard_mtu;
        int                old_rx_urb_size = dev->rx_urb_size;

        // no second zero-length packet read wanted after mtu-sized packets
        if ((ll_mtu % dev->maxpacket) == 0)
                return -EDOM;
        WRITE_ONCE(net->mtu, new_mtu);

        dev->hard_mtu = net->mtu + net->hard_header_len;
        if (dev->rx_urb_size == old_hard_mtu) {
                dev->rx_urb_size = dev->hard_mtu;
                if (dev->rx_urb_size > old_rx_urb_size) {
                        usbnet_pause_rx(dev);
                        usbnet_unlink_rx_urbs(dev);
                        usbnet_resume_rx(dev);
                }
        }

        /* max qlen depend on hard_mtu and rx_urb_size */
        usbnet_update_max_qlen(dev);

        return 0;
}
EXPORT_SYMBOL_GPL(usbnet_change_mtu);

/* The caller must hold list->lock */
static void __usbnet_queue_skb(struct sk_buff_head *list,
                        struct sk_buff *newsk, enum skb_state state)
{
        struct skb_data *entry = (struct skb_data *) newsk->cb;

        __skb_queue_tail(list, newsk);
        entry->state = state;
}

/*-------------------------------------------------------------------------*/

/* some LK 2.4 HCDs oopsed if we freed or resubmitted urbs from
 * completion callbacks.  2.5 should have fixed those bugs...
 */

static enum skb_state defer_bh(struct usbnet *dev, struct sk_buff *skb,
                struct sk_buff_head *list, enum skb_state state)
{
        unsigned long                flags;
        enum skb_state                 old_state;
        struct skb_data *entry = (struct skb_data *) skb->cb;

        spin_lock_irqsave(&list->lock, flags);
        old_state = entry->state;
        entry->state = state;
        __skb_unlink(skb, list);

        /* defer_bh() is never called with list == &dev->done.
         * spin_lock_nested() tells lockdep that it is OK to take
         * dev->done.lock here with list->lock held.
         */
        spin_lock_nested(&dev->done.lock, SINGLE_DEPTH_NESTING);

        __skb_queue_tail(&dev->done, skb);
        if (dev->done.qlen == 1)
                tasklet_schedule(&dev->bh);
        spin_unlock(&dev->done.lock);
        spin_unlock_irqrestore(&list->lock, flags);
        return old_state;
}

/* some work can't be done in tasklets, so we use keventd
 *
 * NOTE:  annoying asymmetry:  if it's active, schedule_work() fails,
 * but tasklet_schedule() doesn't.  hope the failure is rare.
 */
void usbnet_defer_kevent (struct usbnet *dev, int work)
{
        set_bit (work, &dev->flags);
        if (!usbnet_going_away(dev)) {
                if (!schedule_work(&dev->kevent))
                        netdev_dbg(dev->net,
                                   "kevent %s may have been dropped\n",
                                   usbnet_event_names[work]);
                else
                        netdev_dbg(dev->net,
                                   "kevent %s scheduled\n", usbnet_event_names[work]);
        }
}
EXPORT_SYMBOL_GPL(usbnet_defer_kevent);

/*-------------------------------------------------------------------------*/

static void rx_complete (struct urb *urb);

static int rx_submit (struct usbnet *dev, struct urb *urb, gfp_t flags)
{
        struct sk_buff                *skb;
        struct skb_data                *entry;
        int                        retval = 0;
        unsigned long                lockflags;
        size_t                        size = dev->rx_urb_size;

        /* prevent rx skb allocation when error ratio is high */
        if (test_bit(EVENT_RX_KILL, &dev->flags)) {
                usb_free_urb(urb);
                return -ENOLINK;
        }

        if (test_bit(EVENT_NO_IP_ALIGN, &dev->flags))
                skb = __netdev_alloc_skb(dev->net, size, flags);
        else
                skb = __netdev_alloc_skb_ip_align(dev->net, size, flags);
        if (!skb) {
                netif_dbg(dev, rx_err, dev->net, "no rx skb\n");
                usbnet_defer_kevent (dev, EVENT_RX_MEMORY);
                usb_free_urb (urb);
                return -ENOMEM;
        }

        entry = (struct skb_data *) skb->cb;
        entry->urb = urb;
        entry->dev = dev;
        entry->length = 0;

        usb_fill_bulk_urb (urb, dev->udev, dev->in,
                skb->data, size, rx_complete, skb);

        spin_lock_irqsave (&dev->rxq.lock, lockflags);

        if (netif_running (dev->net) &&
            netif_device_present (dev->net) &&
            test_bit(EVENT_DEV_OPEN, &dev->flags) &&
            !test_bit (EVENT_RX_HALT, &dev->flags) &&
            !test_bit (EVENT_DEV_ASLEEP, &dev->flags)) {
                switch (retval = usb_submit_urb (urb, GFP_ATOMIC)) {
                case -EPIPE:
                        usbnet_defer_kevent (dev, EVENT_RX_HALT);
                        break;
                case -ENOMEM:
                        usbnet_defer_kevent (dev, EVENT_RX_MEMORY);
                        break;
                case -ENODEV:
                        netif_dbg(dev, ifdown, dev->net, "device gone\n");
                        netif_device_detach (dev->net);
                        break;
                case -EHOSTUNREACH:
                        retval = -ENOLINK;
                        break;
                default:
                        netif_dbg(dev, rx_err, dev->net,
                                  "rx submit, %d\n", retval);
                        tasklet_schedule (&dev->bh);
                        break;
                case 0:
                        if (!usbnet_going_away(dev))
                                __usbnet_queue_skb(&dev->rxq, skb, rx_start);
                }
        } else {
                netif_dbg(dev, ifdown, dev->net, "rx: stopped\n");
                retval = -ENOLINK;
        }
        spin_unlock_irqrestore (&dev->rxq.lock, lockflags);
        if (retval) {
                dev_kfree_skb_any (skb);
                usb_free_urb (urb);
        }
        return retval;
}


/*-------------------------------------------------------------------------*/

static inline int rx_process(struct usbnet *dev, struct sk_buff *skb)
{
        if (dev->driver_info->rx_fixup &&
            !dev->driver_info->rx_fixup (dev, skb)) {
                /* With RX_ASSEMBLE, rx_fixup() must update counters */
                if (!(dev->driver_info->flags & FLAG_RX_ASSEMBLE))
                        dev->net->stats.rx_errors++;
                return -EPROTO;
        }
        // else network stack removes extra byte if we forced a short packet

        /* all data was already cloned from skb inside the driver */
        if (dev->driver_info->flags & FLAG_MULTI_PACKET)
                return -EALREADY;

        if (skb->len < ETH_HLEN) {
                dev->net->stats.rx_errors++;
                dev->net->stats.rx_length_errors++;
                netif_dbg(dev, rx_err, dev->net, "rx length %d\n", skb->len);
                return -EPROTO;
        }

        usbnet_skb_return(dev, skb);
        return 0;
}

/*-------------------------------------------------------------------------*/

static void rx_complete (struct urb *urb)
{
        struct sk_buff                *skb = (struct sk_buff *) urb->context;
        struct skb_data                *entry = (struct skb_data *) skb->cb;
        struct usbnet                *dev = entry->dev;
        int                        urb_status = urb->status;
        enum skb_state                state;

        skb_put (skb, urb->actual_length);
        state = rx_done;
        entry->urb = NULL;

        switch (urb_status) {
        /* success */
        case 0:
                break;

        /* stalls need manual reset. this is rare ... except that
         * when going through USB 2.0 TTs, unplug appears this way.
         * we avoid the highspeed version of the ETIMEDOUT/EILSEQ
         * storm, recovering as needed.
         */
        case -EPIPE:
                dev->net->stats.rx_errors++;
                usbnet_defer_kevent (dev, EVENT_RX_HALT);
                fallthrough;

        /* software-driven interface shutdown */
        case -ECONNRESET:                /* async unlink */
        case -ESHUTDOWN:                /* hardware gone */
                netif_dbg(dev, ifdown, dev->net,
                          "rx shutdown, code %d\n", urb_status);
                goto block;

        /* we get controller i/o faults during hub_wq disconnect() delays.
         * throttle down resubmits, to avoid log floods; just temporarily,
         * so we still recover when the fault isn't a hub_wq delay.
         */
        case -EPROTO:
        case -ETIME:
        case -EILSEQ:
                dev->net->stats.rx_errors++;
                if (!timer_pending (&dev->delay)) {
                        mod_timer (&dev->delay, jiffies + THROTTLE_JIFFIES);
                        netif_dbg(dev, link, dev->net,
                                  "rx throttle %d\n", urb_status);
                }
block:
                state = rx_cleanup;
                entry->urb = urb;
                urb = NULL;
                break;

        /* data overrun ... flush fifo? */
        case -EOVERFLOW:
                dev->net->stats.rx_over_errors++;
                fallthrough;

        default:
                state = rx_cleanup;
                dev->net->stats.rx_errors++;
                netif_dbg(dev, rx_err, dev->net, "rx status %d\n", urb_status);
                break;
        }

        /* stop rx if packet error rate is high */
        if (++dev->pkt_cnt > 30) {
                dev->pkt_cnt = 0;
                dev->pkt_err = 0;
        } else {
                if (state == rx_cleanup)
                        dev->pkt_err++;
                if (dev->pkt_err > 20)
                        set_bit(EVENT_RX_KILL, &dev->flags);
        }

        state = defer_bh(dev, skb, &dev->rxq, state);

        if (urb) {
                if (netif_running (dev->net) &&
                    !test_bit (EVENT_RX_HALT, &dev->flags) &&
                    state != unlink_start) {
                        rx_submit (dev, urb, GFP_ATOMIC);
                        usb_mark_last_busy(dev->udev);
                        return;
                }
                usb_free_urb (urb);
        }
        netif_dbg(dev, rx_err, dev->net, "no read resubmitted\n");
}

/*-------------------------------------------------------------------------*/
void usbnet_pause_rx(struct usbnet *dev)
{
        set_bit(EVENT_RX_PAUSED, &dev->flags);

        netif_dbg(dev, rx_status, dev->net, "paused rx queue enabled\n");
}
EXPORT_SYMBOL_GPL(usbnet_pause_rx);

void usbnet_resume_rx(struct usbnet *dev)
{
        struct sk_buff *skb;
        int num = 0;

        clear_bit(EVENT_RX_PAUSED, &dev->flags);

        while ((skb = skb_dequeue(&dev->rxq_pause)) != NULL) {
                usbnet_skb_return(dev, skb);
                num++;
        }

        tasklet_schedule(&dev->bh);

        netif_dbg(dev, rx_status, dev->net,
                  "paused rx queue disabled, %d skbs requeued\n", num);
}
EXPORT_SYMBOL_GPL(usbnet_resume_rx);

void usbnet_purge_paused_rxq(struct usbnet *dev)
{
        skb_queue_purge(&dev->rxq_pause);
}
EXPORT_SYMBOL_GPL(usbnet_purge_paused_rxq);

/*-------------------------------------------------------------------------*/

// unlink pending rx/tx; completion handlers do all other cleanup

static int unlink_urbs (struct usbnet *dev, struct sk_buff_head *q)
{
        unsigned long                flags;
        struct sk_buff                *skb;
        int                        count = 0;

        spin_lock_irqsave (&q->lock, flags);
        while (!skb_queue_empty(q)) {
                struct skb_data                *entry;
                struct urb                *urb;
                int                        retval;

                skb_queue_walk(q, skb) {
                        entry = (struct skb_data *) skb->cb;
                        if (entry->state != unlink_start)
                                goto found;
                }
                break;
found:
                entry->state = unlink_start;
                urb = entry->urb;

                /*
                 * Get reference count of the URB to avoid it to be
                 * freed during usb_unlink_urb, which may trigger
                 * use-after-free problem inside usb_unlink_urb since
                 * usb_unlink_urb is always racing with .complete
                 * handler(include defer_bh).
                 */
                usb_get_urb(urb);
                spin_unlock_irqrestore(&q->lock, flags);
                // during some PM-driven resume scenarios,
                // these (async) unlinks complete immediately
                retval = usb_unlink_urb (urb);
                if (retval != -EINPROGRESS && retval != 0)
                        netdev_dbg(dev->net, "unlink urb err, %d\n", retval);
                else
                        count++;
                usb_put_urb(urb);
                spin_lock_irqsave(&q->lock, flags);
        }
        spin_unlock_irqrestore (&q->lock, flags);
        return count;
}

// Flush all pending rx urbs
// minidrivers may need to do this when the MTU changes

void usbnet_unlink_rx_urbs(struct usbnet *dev)
{
        if (netif_running(dev->net)) {
                (void) unlink_urbs (dev, &dev->rxq);
                tasklet_schedule(&dev->bh);
        }
}
EXPORT_SYMBOL_GPL(usbnet_unlink_rx_urbs);

/*-------------------------------------------------------------------------*/

static void wait_skb_queue_empty(struct sk_buff_head *q)
{
        unsigned long flags;

        spin_lock_irqsave(&q->lock, flags);
        while (!skb_queue_empty(q)) {
                spin_unlock_irqrestore(&q->lock, flags);
                schedule_timeout(msecs_to_jiffies(UNLINK_TIMEOUT_MS));
                set_current_state(TASK_UNINTERRUPTIBLE);
                spin_lock_irqsave(&q->lock, flags);
        }
        spin_unlock_irqrestore(&q->lock, flags);
}

// precondition: never called in_interrupt
static void usbnet_terminate_urbs(struct usbnet *dev)
{
        DECLARE_WAITQUEUE(wait, current);
        int temp;

        /* ensure there are no more active urbs */
        add_wait_queue(&dev->wait, &wait);
        set_current_state(TASK_UNINTERRUPTIBLE);
        temp = unlink_urbs(dev, &dev->txq) +
                unlink_urbs(dev, &dev->rxq);

        /* maybe wait for deletions to finish. */
        wait_skb_queue_empty(&dev->rxq);
        wait_skb_queue_empty(&dev->txq);
        wait_skb_queue_empty(&dev->done);
        netif_dbg(dev, ifdown, dev->net,
                  "waited for %d urb completions\n", temp);
        set_current_state(TASK_RUNNING);
        remove_wait_queue(&dev->wait, &wait);
}

int usbnet_stop (struct net_device *net)
{
        struct usbnet                *dev = netdev_priv(net);
        const struct driver_info *info = dev->driver_info;
        int                        retval, pm, mpn;

        clear_bit(EVENT_DEV_OPEN, &dev->flags);
        netif_stop_queue (net);

        netif_info(dev, ifdown, dev->net,
                   "stop stats: rx/tx %lu/%lu, errs %lu/%lu\n",
                   net->stats.rx_packets, net->stats.tx_packets,
                   net->stats.rx_errors, net->stats.tx_errors);

        /* to not race resume */
        pm = usb_autopm_get_interface(dev->intf);
        /* allow minidriver to stop correctly (wireless devices to turn off
         * radio etc) */
        if (info->stop) {
                retval = info->stop(dev);
                if (retval < 0)
                        netif_info(dev, ifdown, dev->net,
                                   "stop fail (%d) usbnet usb-%s-%s, %s\n",
                                   retval,
                                   dev->udev->bus->bus_name, dev->udev->devpath,
                                   info->description);
        }

        if (!(info->flags & FLAG_AVOID_UNLINK_URBS))
                usbnet_terminate_urbs(dev);

        usbnet_status_stop(dev);

        usbnet_purge_paused_rxq(dev);

        mpn = !test_and_clear_bit(EVENT_NO_RUNTIME_PM, &dev->flags);

        /* deferred work (timer, softirq, task) must also stop */
        dev->flags = 0;
        del_timer_sync(&dev->delay);
        tasklet_kill(&dev->bh);
        cancel_work_sync(&dev->kevent);

        /* We have cyclic dependencies. Those calls are needed
         * to break a cycle. We cannot fall into the gaps because
         * we have a flag
         */
        tasklet_kill(&dev->bh);
        del_timer_sync(&dev->delay);
        cancel_work_sync(&dev->kevent);

        if (!pm)
                usb_autopm_put_interface(dev->intf);

        if (info->manage_power && mpn)
                info->manage_power(dev, 0);
        else
                usb_autopm_put_interface(dev->intf);

        return 0;
}
EXPORT_SYMBOL_GPL(usbnet_stop);

/*-------------------------------------------------------------------------*/

// posts reads, and enables write queuing

// precondition: never called in_interrupt

int usbnet_open (struct net_device *net)
{
        struct usbnet                *dev = netdev_priv(net);
        int                        retval;
        const struct driver_info *info = dev->driver_info;

        if ((retval = usb_autopm_get_interface(dev->intf)) < 0) {
                netif_info(dev, ifup, dev->net,
                           "resumption fail (%d) usbnet usb-%s-%s, %s\n",
                           retval,
                           dev->udev->bus->bus_name,
                           dev->udev->devpath,
                           info->description);
                goto done_nopm;
        }

        // put into "known safe" state
        if (info->reset && (retval = info->reset (dev)) < 0) {
                netif_info(dev, ifup, dev->net,
                           "open reset fail (%d) usbnet usb-%s-%s, %s\n",
                           retval,
                           dev->udev->bus->bus_name,
                           dev->udev->devpath,
                           info->description);
                goto done;
        }

        /* hard_mtu or rx_urb_size may change in reset() */
        usbnet_update_max_qlen(dev);

        // insist peer be connected
        if (info->check_connect && (retval = info->check_connect (dev)) < 0) {
                netif_err(dev, ifup, dev->net, "can't open; %d\n", retval);
                goto done;
        }

        /* start any status interrupt transfer */
        if (dev->interrupt) {
                retval = usbnet_status_start(dev, GFP_KERNEL);
                if (retval < 0) {
                        netif_err(dev, ifup, dev->net,
                                  "intr submit %d\n", retval);
                        goto done;
                }
        }

        set_bit(EVENT_DEV_OPEN, &dev->flags);
        netif_start_queue (net);
        netif_info(dev, ifup, dev->net,
                   "open: enable queueing (rx %d, tx %d) mtu %d %s framing\n",
                   (int)RX_QLEN(dev), (int)TX_QLEN(dev),
                   dev->net->mtu,
                   (dev->driver_info->flags & FLAG_FRAMING_NC) ? "NetChip" :
                   (dev->driver_info->flags & FLAG_FRAMING_GL) ? "GeneSys" :
                   (dev->driver_info->flags & FLAG_FRAMING_Z) ? "Zaurus" :
                   (dev->driver_info->flags & FLAG_FRAMING_RN) ? "RNDIS" :
                   (dev->driver_info->flags & FLAG_FRAMING_AX) ? "ASIX" :
                   "simple");

        /* reset rx error state */
        dev->pkt_cnt = 0;
        dev->pkt_err = 0;
        clear_bit(EVENT_RX_KILL, &dev->flags);

        // delay posting reads until we're fully open
        tasklet_schedule (&dev->bh);
        if (info->manage_power) {
                retval = info->manage_power(dev, 1);
                if (retval < 0) {
                        retval = 0;
                        set_bit(EVENT_NO_RUNTIME_PM, &dev->flags);
                } else {
                        usb_autopm_put_interface(dev->intf);
                }
        }
        return retval;
done:
        usb_autopm_put_interface(dev->intf);
done_nopm:
        return retval;
}
EXPORT_SYMBOL_GPL(usbnet_open);

/*-------------------------------------------------------------------------*/

/* ethtool methods; minidrivers may need to add some more, but
 * they'll probably want to use this base set.
 */

/* These methods are written on the assumption that the device
 * uses MII
 */
int usbnet_get_link_ksettings_mii(struct net_device *net,
                              struct ethtool_link_ksettings *cmd)
{
        struct usbnet *dev = netdev_priv(net);

        if (!dev->mii.mdio_read)
                return -EOPNOTSUPP;

        mii_ethtool_get_link_ksettings(&dev->mii, cmd);

        return 0;
}
EXPORT_SYMBOL_GPL(usbnet_get_link_ksettings_mii);

int usbnet_get_link_ksettings_internal(struct net_device *net,
                                        struct ethtool_link_ksettings *cmd)
{
        struct usbnet *dev = netdev_priv(net);

        /* the assumption that speed is equal on tx and rx
         * is deeply engrained into the networking layer.
         * For wireless stuff it is not true.
         * We assume that rx_speed matters more.
         */
        if (dev->rx_speed != SPEED_UNSET)
                cmd->base.speed = dev->rx_speed / 1000000;
        else if (dev->tx_speed != SPEED_UNSET)
                cmd->base.speed = dev->tx_speed / 1000000;
        else
                cmd->base.speed = SPEED_UNKNOWN;

        return 0;
}
EXPORT_SYMBOL_GPL(usbnet_get_link_ksettings_internal);

int usbnet_set_link_ksettings_mii(struct net_device *net,
                              const struct ethtool_link_ksettings *cmd)
{
        struct usbnet *dev = netdev_priv(net);
        int retval;

        if (!dev->mii.mdio_write)
                return -EOPNOTSUPP;

        retval = mii_ethtool_set_link_ksettings(&dev->mii, cmd);

        /* link speed/duplex might have changed */
        if (dev->driver_info->link_reset)
                dev->driver_info->link_reset(dev);

        /* hard_mtu or rx_urb_size may change in link_reset() */
        usbnet_update_max_qlen(dev);

        return retval;
}
EXPORT_SYMBOL_GPL(usbnet_set_link_ksettings_mii);

u32 usbnet_get_link (struct net_device *net)
{
        struct usbnet *dev = netdev_priv(net);

        /* If a check_connect is defined, return its result */
        if (dev->driver_info->check_connect)
                return dev->driver_info->check_connect (dev) == 0;

        /* if the device has mii operations, use those */
        if (dev->mii.mdio_read)
                return mii_link_ok(&dev->mii);

        /* Otherwise, dtrt for drivers calling netif_carrier_{on,off} */
        return ethtool_op_get_link(net);
}
EXPORT_SYMBOL_GPL(usbnet_get_link);

int usbnet_nway_reset(struct net_device *net)
{
        struct usbnet *dev = netdev_priv(net);

        if (!dev->mii.mdio_write)
                return -EOPNOTSUPP;

        return mii_nway_restart(&dev->mii);
}
EXPORT_SYMBOL_GPL(usbnet_nway_reset);

void usbnet_get_drvinfo (struct net_device *net, struct ethtool_drvinfo *info)
{
        struct usbnet *dev = netdev_priv(net);

        strscpy(info->driver, dev->driver_name, sizeof(info->driver));
        strscpy(info->fw_version, dev->driver_info->description,
                sizeof(info->fw_version));
        usb_make_path (dev->udev, info->bus_info, sizeof info->bus_info);
}
EXPORT_SYMBOL_GPL(usbnet_get_drvinfo);

u32 usbnet_get_msglevel (struct net_device *net)
{
        struct usbnet *dev = netdev_priv(net);

        return dev->msg_enable;
}
EXPORT_SYMBOL_GPL(usbnet_get_msglevel);

void usbnet_set_msglevel (struct net_device *net, u32 level)
{
        struct usbnet *dev = netdev_priv(net);

        dev->msg_enable = level;
}
EXPORT_SYMBOL_GPL(usbnet_set_msglevel);

/* drivers may override default ethtool_ops in their bind() routine */
static const struct ethtool_ops usbnet_ethtool_ops = {
        .get_link                = usbnet_get_link,
        .nway_reset                = usbnet_nway_reset,
        .get_drvinfo                = usbnet_get_drvinfo,
        .get_msglevel                = usbnet_get_msglevel,
        .set_msglevel                = usbnet_set_msglevel,
        .get_ts_info                = ethtool_op_get_ts_info,
        .get_link_ksettings        = usbnet_get_link_ksettings_mii,
        .set_link_ksettings        = usbnet_set_link_ksettings_mii,
};

/*-------------------------------------------------------------------------*/

static void __handle_link_change(struct usbnet *dev)
{
        if (!test_bit(EVENT_DEV_OPEN, &dev->flags))
                return;

        if (!netif_carrier_ok(dev->net)) {
                /* kill URBs for reading packets to save bus bandwidth */
                unlink_urbs(dev, &dev->rxq);

                /*
                 * tx_timeout will unlink URBs for sending packets and
                 * tx queue is stopped by netcore after link becomes off
                 */
        } else {
                /* submitting URBs for reading packets */
                tasklet_schedule(&dev->bh);
        }

        /* hard_mtu or rx_urb_size may change during link change */
        usbnet_update_max_qlen(dev);

        clear_bit(EVENT_LINK_CHANGE, &dev->flags);
}

void usbnet_set_rx_mode(struct net_device *net)
{
        struct usbnet                *dev = netdev_priv(net);

        usbnet_defer_kevent(dev, EVENT_SET_RX_MODE);
}
EXPORT_SYMBOL_GPL(usbnet_set_rx_mode);

static void __handle_set_rx_mode(struct usbnet *dev)
{
        if (dev->driver_info->set_rx_mode)
                (dev->driver_info->set_rx_mode)(dev);

        clear_bit(EVENT_SET_RX_MODE, &dev->flags);
}

/* work that cannot be done in interrupt context uses keventd.
 *
 * NOTE:  with 2.5 we could do more of this using completion callbacks,
 * especially now that control transfers can be queued.
 */
static void
usbnet_deferred_kevent (struct work_struct *work)
{
        struct usbnet                *dev =
                container_of(work, struct usbnet, kevent);
        int                        status;

        /* usb_clear_halt() needs a thread context */
        if (test_bit (EVENT_TX_HALT, &dev->flags)) {
                unlink_urbs (dev, &dev->txq);
                status = usb_autopm_get_interface(dev->intf);
                if (status < 0)
                        goto fail_pipe;
                status = usb_clear_halt (dev->udev, dev->out);
                usb_autopm_put_interface(dev->intf);
                if (status < 0 &&
                    status != -EPIPE &&
                    status != -ESHUTDOWN) {
                        if (netif_msg_tx_err (dev))
fail_pipe:
                                netdev_err(dev->net, "can't clear tx halt, status %d\n",
                                           status);
                } else {
                        clear_bit (EVENT_TX_HALT, &dev->flags);
                        if (status != -ESHUTDOWN)
                                netif_wake_queue (dev->net);
                }
        }
        if (test_bit (EVENT_RX_HALT, &dev->flags)) {
                unlink_urbs (dev, &dev->rxq);
                status = usb_autopm_get_interface(dev->intf);
                if (status < 0)
                        goto fail_halt;
                status = usb_clear_halt (dev->udev, dev->in);
                usb_autopm_put_interface(dev->intf);
                if (status < 0 &&
                    status != -EPIPE &&
                    status != -ESHUTDOWN) {
                        if (netif_msg_rx_err (dev))
fail_halt:
                                netdev_err(dev->net, "can't clear rx halt, status %d\n",
                                           status);
                } else {
                        clear_bit (EVENT_RX_HALT, &dev->flags);
                        if (!usbnet_going_away(dev))
                                tasklet_schedule(&dev->bh);
                }
        }

        /* tasklet could resubmit itself forever if memory is tight */
        if (test_bit (EVENT_RX_MEMORY, &dev->flags)) {
                struct urb        *urb = NULL;
                int resched = 1;

                if (netif_running (dev->net))
                        urb = usb_alloc_urb (0, GFP_KERNEL);
                else
                        clear_bit (EVENT_RX_MEMORY, &dev->flags);
                if (urb != NULL) {
                        clear_bit (EVENT_RX_MEMORY, &dev->flags);
                        status = usb_autopm_get_interface(dev->intf);
                        if (status < 0) {
                                usb_free_urb(urb);
                                goto fail_lowmem;
                        }
                        if (rx_submit (dev, urb, GFP_KERNEL) == -ENOLINK)
                                resched = 0;
                        usb_autopm_put_interface(dev->intf);
fail_lowmem:
                        if (resched)
                                if (!usbnet_going_away(dev))
                                        tasklet_schedule(&dev->bh);
                }
        }

        if (test_bit (EVENT_LINK_RESET, &dev->flags)) {
                const struct driver_info *info = dev->driver_info;
                int                        retval = 0;

                clear_bit (EVENT_LINK_RESET, &dev->flags);
                status = usb_autopm_get_interface(dev->intf);
                if (status < 0)
                        goto skip_reset;
                if(info->link_reset && (retval = info->link_reset(dev)) < 0) {
                        usb_autopm_put_interface(dev->intf);
skip_reset:
                        netdev_info(dev->net, "link reset failed (%d) usbnet usb-%s-%s, %s\n",
                                    retval,
                                    dev->udev->bus->bus_name,
                                    dev->udev->devpath,
                                    info->description);
                } else {
                        usb_autopm_put_interface(dev->intf);
                }

                /* handle link change from link resetting */
                __handle_link_change(dev);
        }

        if (test_bit (EVENT_LINK_CHANGE, &dev->flags))
                __handle_link_change(dev);

        if (test_bit (EVENT_SET_RX_MODE, &dev->flags))
                __handle_set_rx_mode(dev);


        if (dev->flags)
                netdev_dbg(dev->net, "kevent done, flags = 0x%lx\n", dev->flags);
}

/*-------------------------------------------------------------------------*/

static void tx_complete (struct urb *urb)
{
        struct sk_buff                *skb = (struct sk_buff *) urb->context;
        struct skb_data                *entry = (struct skb_data *) skb->cb;
        struct usbnet                *dev = entry->dev;

        if (urb->status == 0) {
                struct pcpu_sw_netstats *stats64 = this_cpu_ptr(dev->net->tstats);
                unsigned long flags;

                flags = u64_stats_update_begin_irqsave(&stats64->syncp);
                u64_stats_add(&stats64->tx_packets, entry->packets);
                u64_stats_add(&stats64->tx_bytes, entry->length);
                u64_stats_update_end_irqrestore(&stats64->syncp, flags);
        } else {
                dev->net->stats.tx_errors++;

                switch (urb->status) {
                case -EPIPE:
                        usbnet_defer_kevent (dev, EVENT_TX_HALT);
                        break;

                /* software-driven interface shutdown */
                case -ECONNRESET:                // async unlink
                case -ESHUTDOWN:                // hardware gone
                        break;

                /* like rx, tx gets controller i/o faults during hub_wq
                 * delays and so it uses the same throttling mechanism.
                 */
                case -EPROTO:
                case -ETIME:
                case -EILSEQ:
                        usb_mark_last_busy(dev->udev);
                        if (!timer_pending (&dev->delay)) {
                                mod_timer (&dev->delay,
                                        jiffies + THROTTLE_JIFFIES);
                                netif_dbg(dev, link, dev->net,
                                          "tx throttle %d\n", urb->status);
                        }
                        netif_stop_queue (dev->net);
                        break;
                default:
                        netif_dbg(dev, tx_err, dev->net,
                                  "tx err %d\n", entry->urb->status);
                        break;
                }
        }

        usb_autopm_put_interface_async(dev->intf);
        (void) defer_bh(dev, skb, &dev->txq, tx_done);
}

/*-------------------------------------------------------------------------*/

void usbnet_tx_timeout (struct net_device *net, unsigned int txqueue)
{
        struct usbnet                *dev = netdev_priv(net);

        unlink_urbs (dev, &dev->txq);
        tasklet_schedule (&dev->bh);
        /* this needs to be handled individually because the generic layer
         * doesn't know what is sufficient and could not restore private
         * information if a remedy of an unconditional reset were used.
         */
        if (dev->driver_info->recover)
                (dev->driver_info->recover)(dev);
}
EXPORT_SYMBOL_GPL(usbnet_tx_timeout);

/*-------------------------------------------------------------------------*/

static int build_dma_sg(const struct sk_buff *skb, struct urb *urb)
{
        unsigned num_sgs, total_len = 0;
        int i, s = 0;

        num_sgs = skb_shinfo(skb)->nr_frags + 1;
        if (num_sgs == 1)
                return 0;

        /* reserve one for zero packet */
        urb->sg = kmalloc_array(num_sgs + 1, sizeof(struct scatterlist),
                                GFP_ATOMIC);
        if (!urb->sg)
                return -ENOMEM;

        urb->num_sgs = num_sgs;
        sg_init_table(urb->sg, urb->num_sgs + 1);

        sg_set_buf(&urb->sg[s++], skb->data, skb_headlen(skb));
        total_len += skb_headlen(skb);

        for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
                skb_frag_t *f = &skb_shinfo(skb)->frags[i];

                total_len += skb_frag_size(f);
                sg_set_page(&urb->sg[i + s], skb_frag_page(f), skb_frag_size(f),
                            skb_frag_off(f));
        }
        urb->transfer_buffer_length = total_len;

        return 1;
}

netdev_tx_t usbnet_start_xmit (struct sk_buff *skb,
                                     struct net_device *net)
{
        struct usbnet                *dev = netdev_priv(net);
        unsigned int                        length;
        struct urb                *urb = NULL;
        struct skb_data                *entry;
        const struct driver_info *info = dev->driver_info;
        unsigned long                flags;
        int retval;

        if (skb)
                skb_tx_timestamp(skb);

        // some devices want funky USB-level framing, for
        // win32 driver (usually) and/or hardware quirks
        if (info->tx_fixup) {
                skb = info->tx_fixup (dev, skb, GFP_ATOMIC);
                if (!skb) {
                        /* packet collected; minidriver waiting for more */
                        if (info->flags & FLAG_MULTI_PACKET)
                                goto not_drop;
                        netif_dbg(dev, tx_err, dev->net, "can't tx_fixup skb\n");
                        goto drop;
                }
        }

        if (!(urb = usb_alloc_urb (0, GFP_ATOMIC))) {
                netif_dbg(dev, tx_err, dev->net, "no urb\n");
                goto drop;
        }

        entry = (struct skb_data *) skb->cb;
        entry->urb = urb;
        entry->dev = dev;

        usb_fill_bulk_urb (urb, dev->udev, dev->out,
                        skb->data, skb->len, tx_complete, skb);
        if (dev->can_dma_sg) {
                if (build_dma_sg(skb, urb) < 0)
                        goto drop;
        }
        length = urb->transfer_buffer_length;

        /* don't assume the hardware handles USB_ZERO_PACKET
         * NOTE:  strictly conforming cdc-ether devices should expect
         * the ZLP here, but ignore the one-byte packet.
         * NOTE2: CDC NCM specification is different from CDC ECM when
         * handling ZLP/short packets, so cdc_ncm driver will make short
         * packet itself if needed.
         */
        if (length % dev->maxpacket == 0) {
                if (!(info->flags & FLAG_SEND_ZLP)) {
                        if (!(info->flags & FLAG_MULTI_PACKET)) {
                                length++;
                                if (skb_tailroom(skb) && !urb->num_sgs) {
                                        skb->data[skb->len] = 0;
                                        __skb_put(skb, 1);
                                } else if (urb->num_sgs)
                                        sg_set_buf(&urb->sg[urb->num_sgs++],
                                                        dev->padding_pkt, 1);
                        }
                } else
                        urb->transfer_flags |= URB_ZERO_PACKET;
        }
        urb->transfer_buffer_length = length;

        if (info->flags & FLAG_MULTI_PACKET) {
                /* Driver has set number of packets and a length delta.
                 * Calculate the complete length and ensure that it's
                 * positive.
                 */
                entry->length += length;
                if (WARN_ON_ONCE(entry->length <= 0))
                        entry->length = length;
        } else {
                usbnet_set_skb_tx_stats(skb, 1, length);
        }

        spin_lock_irqsave(&dev->txq.lock, flags);
        retval = usb_autopm_get_interface_async(dev->intf);
        if (retval < 0) {
                spin_unlock_irqrestore(&dev->txq.lock, flags);
                goto drop;
        }
        if (netif_queue_stopped(net)) {
                usb_autopm_put_interface_async(dev->intf);
                spin_unlock_irqrestore(&dev->txq.lock, flags);
                goto drop;
        }

#ifdef CONFIG_PM
        /* if this triggers the device is still a sleep */
        if (test_bit(EVENT_DEV_ASLEEP, &dev->flags)) {
                /* transmission will be done in resume */
                usb_anchor_urb(urb, &dev->deferred);
                /* no use to process more packets */
                netif_stop_queue(net);
                usb_put_urb(urb);
                spin_unlock_irqrestore(&dev->txq.lock, flags);
                netdev_dbg(dev->net, "Delaying transmission for resumption\n");
                goto deferred;
        }
#endif

        switch ((retval = usb_submit_urb (urb, GFP_ATOMIC))) {
        case -EPIPE:
                netif_stop_queue (net);
                usbnet_defer_kevent (dev, EVENT_TX_HALT);
                usb_autopm_put_interface_async(dev->intf);
                break;
        default:
                usb_autopm_put_interface_async(dev->intf);
                netif_dbg(dev, tx_err, dev->net,
                          "tx: submit urb err %d\n", retval);
                break;
        case 0:
                netif_trans_update(net);
                __usbnet_queue_skb(&dev->txq, skb, tx_start);
                if (dev->txq.qlen >= TX_QLEN (dev))
                        netif_stop_queue (net);
        }
        spin_unlock_irqrestore (&dev->txq.lock, flags);

        if (retval) {
                netif_dbg(dev, tx_err, dev->net, "drop, code %d\n", retval);
drop:
                dev->net->stats.tx_dropped++;
not_drop:
                if (skb)
                        dev_kfree_skb_any (skb);
                if (urb) {
                        kfree(urb->sg);
                        usb_free_urb(urb);
                }
        } else
                netif_dbg(dev, tx_queued, dev->net,
                          "> tx, len %u, type 0x%x\n", length, skb->protocol);
#ifdef CONFIG_PM
deferred:
#endif
        return NETDEV_TX_OK;
}
EXPORT_SYMBOL_GPL(usbnet_start_xmit);

static int rx_alloc_submit(struct usbnet *dev, gfp_t flags)
{
        struct urb        *urb;
        int                i;
        int                ret = 0;

        /* don't refill the queue all at once */
        for (i = 0; i < 10 && dev->rxq.qlen < RX_QLEN(dev); i++) {
                urb = usb_alloc_urb(0, flags);
                if (urb != NULL) {
                        ret = rx_submit(dev, urb, flags);
                        if (ret)
                                goto err;
                } else {
                        ret = -ENOMEM;
                        goto err;
                }
        }
err:
        return ret;
}

static inline void usb_free_skb(struct sk_buff *skb)
{
        struct skb_data *entry = (struct skb_data *)skb->cb;

        usb_free_urb(entry->urb);
        dev_kfree_skb(skb);
}

/*-------------------------------------------------------------------------*/

// tasklet (work deferred from completions, in_irq) or timer

static void usbnet_bh (struct timer_list *t)
{
        struct usbnet                *dev = from_timer(dev, t, delay);
        struct sk_buff                *skb;
        struct skb_data                *entry;

        while ((skb = skb_dequeue (&dev->done))) {
                entry = (struct skb_data *) skb->cb;
                switch (entry->state) {
                case rx_done:
                        if (rx_process(dev, skb))
                                usb_free_skb(skb);
                        continue;
                case tx_done:
                        kfree(entry->urb->sg);
                        fallthrough;
                case rx_cleanup:
                        usb_free_skb(skb);
                        continue;
                default:
                        netdev_dbg(dev->net, "bogus skb state %d\n", entry->state);
                }
        }

        /* restart RX again after disabling due to high error rate */
        clear_bit(EVENT_RX_KILL, &dev->flags);

        /* waiting for all pending urbs to complete?
         * only then can we forgo submitting anew
         */
        if (waitqueue_active(&dev->wait)) {
                if (dev->txq.qlen + dev->rxq.qlen + dev->done.qlen == 0)
                        wake_up_all(&dev->wait);

        // or are we maybe short a few urbs?
        } else if (netif_running (dev->net) &&
                   netif_device_present (dev->net) &&
                   netif_carrier_ok(dev->net) &&
                   !usbnet_going_away(dev) &&
                   !timer_pending(&dev->delay) &&
                   !test_bit(EVENT_RX_PAUSED, &dev->flags) &&
                   !test_bit(EVENT_RX_HALT, &dev->flags)) {
                int        temp = dev->rxq.qlen;

                if (temp < RX_QLEN(dev)) {
                        if (rx_alloc_submit(dev, GFP_ATOMIC) == -ENOLINK)
                                return;
                        if (temp != dev->rxq.qlen)
                                netif_dbg(dev, link, dev->net,
                                          "rxqlen %d --> %d\n",
                                          temp, dev->rxq.qlen);
                        if (dev->rxq.qlen < RX_QLEN(dev))
                                tasklet_schedule (&dev->bh);
                }
                if (dev->txq.qlen < TX_QLEN (dev))
                        netif_wake_queue (dev->net);
        }
}

static void usbnet_bh_tasklet(struct tasklet_struct *t)
{
        struct usbnet *dev = from_tasklet(dev, t, bh);

        usbnet_bh(&dev->delay);
}


/*-------------------------------------------------------------------------
 *
 * USB Device Driver support
 *
 *-------------------------------------------------------------------------*/

// precondition: never called in_interrupt

void usbnet_disconnect (struct usb_interface *intf)
{
        struct usbnet                *dev;
        struct usb_device        *xdev;
        struct net_device        *net;
        struct urb                *urb;

        dev = usb_get_intfdata(intf);
        usb_set_intfdata(intf, NULL);
        if (!dev)
                return;
        usbnet_mark_going_away(dev);

        xdev = interface_to_usbdev (intf);

        netif_info(dev, probe, dev->net, "unregister '%s' usb-%s-%s, %s\n",
                   intf->dev.driver->name,
                   xdev->bus->bus_name, xdev->devpath,
                   dev->driver_info->description);

        net = dev->net;
        unregister_netdev (net);

        while ((urb = usb_get_from_anchor(&dev->deferred))) {
                dev_kfree_skb(urb->context);
                kfree(urb->sg);
                usb_free_urb(urb);
        }

        if (dev->driver_info->unbind)
                dev->driver_info->unbind(dev, intf);

        usb_kill_urb(dev->interrupt);
        usb_free_urb(dev->interrupt);
        kfree(dev->padding_pkt);

        free_netdev(net);
}
EXPORT_SYMBOL_GPL(usbnet_disconnect);

static const struct net_device_ops usbnet_netdev_ops = {
        .ndo_open                = usbnet_open,
        .ndo_stop                = usbnet_stop,
        .ndo_start_xmit                = usbnet_start_xmit,
        .ndo_tx_timeout                = usbnet_tx_timeout,
        .ndo_set_rx_mode        = usbnet_set_rx_mode,
        .ndo_change_mtu                = usbnet_change_mtu,
        .ndo_set_mac_address         = eth_mac_addr,
        .ndo_validate_addr        = eth_validate_addr,
};

/*-------------------------------------------------------------------------*/

// precondition: never called in_interrupt

static const struct device_type wlan_type = {
        .name        = "wlan",
};

static const struct device_type wwan_type = {
        .name        = "wwan",
};

int
usbnet_probe (struct usb_interface *udev, const struct usb_device_id *prod)
{
        struct usbnet                        *dev;
        struct net_device                *net;
        struct usb_host_interface        *interface;
        const struct driver_info        *info;
        struct usb_device                *xdev;
        int                                status;
        const char                        *name;
        struct usb_driver         *driver = to_usb_driver(udev->dev.driver);

        /* usbnet already took usb runtime pm, so have to enable the feature
         * for usb interface, otherwise usb_autopm_get_interface may return
         * failure if RUNTIME_PM is enabled.
         */
        if (!driver->supports_autosuspend) {
                driver->supports_autosuspend = 1;
                pm_runtime_enable(&udev->dev);
        }

        name = udev->dev.driver->name;
        info = (const struct driver_info *) prod->driver_info;
        if (!info) {
                dev_dbg (&udev->dev, "blacklisted by %s\n", name);
                return -ENODEV;
        }
        xdev = interface_to_usbdev (udev);
        interface = udev->cur_altsetting;

        status = -ENOMEM;

        // set up our own records
        net = alloc_etherdev(sizeof(*dev));
        if (!net)
                goto out;

        /* netdev_printk() needs this so do it as early as possible */
        SET_NETDEV_DEV(net, &udev->dev);

        dev = netdev_priv(net);
        dev->udev = xdev;
        dev->intf = udev;
        dev->driver_info = info;
        dev->driver_name = name;
        dev->rx_speed = SPEED_UNSET;
        dev->tx_speed = SPEED_UNSET;

        dev->msg_enable = netif_msg_init (msg_level, NETIF_MSG_DRV
                                | NETIF_MSG_PROBE | NETIF_MSG_LINK);
        init_waitqueue_head(&dev->wait);
        skb_queue_head_init (&dev->rxq);
        skb_queue_head_init (&dev->txq);
        skb_queue_head_init (&dev->done);
        skb_queue_head_init(&dev->rxq_pause);
        tasklet_setup(&dev->bh, usbnet_bh_tasklet);
        INIT_WORK (&dev->kevent, usbnet_deferred_kevent);
        init_usb_anchor(&dev->deferred);
        timer_setup(&dev->delay, usbnet_bh, 0);
        mutex_init (&dev->phy_mutex);
        mutex_init(&dev->interrupt_mutex);
        dev->interrupt_count = 0;

        dev->net = net;
        strscpy(net->name, "usb%d", sizeof(net->name));

        /* rx and tx sides can use different message sizes;
         * bind() should set rx_urb_size in that case.
         */
        dev->hard_mtu = net->mtu + net->hard_header_len;
        net->min_mtu = 0;
        net->max_mtu = ETH_MAX_MTU;
        net->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;

        net->netdev_ops = &usbnet_netdev_ops;
        net->watchdog_timeo = TX_TIMEOUT_JIFFIES;
        net->ethtool_ops = &usbnet_ethtool_ops;
        net->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;

        // allow device-specific bind/init procedures
        // NOTE net->name still not usable ...
        if (info->bind) {
                status = info->bind (dev, udev);
                if (status < 0)
                        goto out1;

                // heuristic:  "usb%d" for links we know are two-host,
                // else "eth%d" when there's reasonable doubt.  userspace
                // can rename the link if it knows better.
                if ((dev->driver_info->flags & FLAG_ETHER) != 0 &&
                    ((dev->driver_info->flags & FLAG_POINTTOPOINT) == 0 ||
                     (net->dev_addr [0] & 0x02) == 0))
                        strscpy(net->name, "eth%d", sizeof(net->name));
                /* WLAN devices should always be named "wlan%d" */
                if ((dev->driver_info->flags & FLAG_WLAN) != 0)
                        strscpy(net->name, "wlan%d", sizeof(net->name));
                /* WWAN devices should always be named "wwan%d" */
                if ((dev->driver_info->flags & FLAG_WWAN) != 0)
                        strscpy(net->name, "wwan%d", sizeof(net->name));

                /* devices that cannot do ARP */
                if ((dev->driver_info->flags & FLAG_NOARP) != 0)
                        net->flags |= IFF_NOARP;

                /* maybe the remote can't receive an Ethernet MTU */
                if (net->mtu > (dev->hard_mtu - net->hard_header_len))
                        net->mtu = dev->hard_mtu - net->hard_header_len;
        } else if (!info->in || !info->out)
                status = usbnet_get_endpoints (dev, udev);
        else {
                u8 ep_addrs[3] = {
                        info->in + USB_DIR_IN, info->out + USB_DIR_OUT, 0
                };

                dev->in = usb_rcvbulkpipe (xdev, info->in);
                dev->out = usb_sndbulkpipe (xdev, info->out);
                if (!(info->flags & FLAG_NO_SETINT))
                        status = usb_set_interface (xdev,
                                interface->desc.bInterfaceNumber,
                                interface->desc.bAlternateSetting);
                else
                        status = 0;

                if (status == 0 && !usb_check_bulk_endpoints(udev, ep_addrs))
                        status = -EINVAL;
        }
        if (status >= 0 && dev->status)
                status = init_status (dev, udev);
        if (status < 0)
                goto out3;

        if (!dev->rx_urb_size)
                dev->rx_urb_size = dev->hard_mtu;
        dev->maxpacket = usb_maxpacket(dev->udev, dev->out);
        if (dev->maxpacket == 0) {
                /* that is a broken device */
                status = -ENODEV;
                goto out4;
        }

        /* this flags the device for user space */
        if (!is_valid_ether_addr(net->dev_addr))
                eth_hw_addr_random(net);

        if ((dev->driver_info->flags & FLAG_WLAN) != 0)
                SET_NETDEV_DEVTYPE(net, &wlan_type);
        if ((dev->driver_info->flags & FLAG_WWAN) != 0)
                SET_NETDEV_DEVTYPE(net, &wwan_type);

        /* initialize max rx_qlen and tx_qlen */
        usbnet_update_max_qlen(dev);

        if (dev->can_dma_sg && !(info->flags & FLAG_SEND_ZLP) &&
                !(info->flags & FLAG_MULTI_PACKET)) {
                dev->padding_pkt = kzalloc(1, GFP_KERNEL);
                if (!dev->padding_pkt) {
                        status = -ENOMEM;
                        goto out4;
                }
        }

        status = register_netdev (net);
        if (status)
                goto out5;
        netif_info(dev, probe, dev->net,
                   "register '%s' at usb-%s-%s, %s, %pM\n",
                   udev->dev.driver->name,
                   xdev->bus->bus_name, xdev->devpath,
                   dev->driver_info->description,
                   net->dev_addr);

        // ok, it's ready to go.
        usb_set_intfdata (udev, dev);

        netif_device_attach (net);

        if (dev->driver_info->flags & FLAG_LINK_INTR)
                usbnet_link_change(dev, 0, 0);

        return 0;

out5:
        kfree(dev->padding_pkt);
out4:
        usb_free_urb(dev->interrupt);
out3:
        if (info->unbind)
                info->unbind (dev, udev);
out1:
        /* subdrivers must undo all they did in bind() if they
         * fail it, but we may fail later and a deferred kevent
         * may trigger an error resubmitting itself and, worse,
         * schedule a timer. So we kill it all just in case.
         */
        cancel_work_sync(&dev->kevent);
        del_timer_sync(&dev->delay);
        free_netdev(net);
out:
        return status;
}
EXPORT_SYMBOL_GPL(usbnet_probe);

/*-------------------------------------------------------------------------*/

/*
 * suspend the whole driver as soon as the first interface is suspended
 * resume only when the last interface is resumed
 */

int usbnet_suspend (struct usb_interface *intf, pm_message_t message)
{
        struct usbnet                *dev = usb_get_intfdata(intf);

        if (!dev->suspend_count++) {
                spin_lock_irq(&dev->txq.lock);
                /* don't autosuspend while transmitting */
                if (dev->txq.qlen && PMSG_IS_AUTO(message)) {
                        dev->suspend_count--;
                        spin_unlock_irq(&dev->txq.lock);
                        return -EBUSY;
                } else {
                        set_bit(EVENT_DEV_ASLEEP, &dev->flags);
                        spin_unlock_irq(&dev->txq.lock);
                }
                /*
                 * accelerate emptying of the rx and queues, to avoid
                 * having everything error out.
                 */
                netif_device_detach (dev->net);
                usbnet_terminate_urbs(dev);
                __usbnet_status_stop_force(dev);

                /*
                 * reattach so runtime management can use and
                 * wake the device
                 */
                netif_device_attach (dev->net);
        }
        return 0;
}
EXPORT_SYMBOL_GPL(usbnet_suspend);

int usbnet_resume (struct usb_interface *intf)
{
        struct usbnet                *dev = usb_get_intfdata(intf);
        struct sk_buff          *skb;
        struct urb              *res;
        int                     retval;

        if (!--dev->suspend_count) {
                /* resume interrupt URB if it was previously submitted */
                __usbnet_status_start_force(dev, GFP_NOIO);

                spin_lock_irq(&dev->txq.lock);
                while ((res = usb_get_from_anchor(&dev->deferred))) {

                        skb = (struct sk_buff *)res->context;
                        retval = usb_submit_urb(res, GFP_ATOMIC);
                        if (retval < 0) {
                                dev_kfree_skb_any(skb);
                                kfree(res->sg);
                                usb_free_urb(res);
                                usb_autopm_put_interface_async(dev->intf);
                        } else {
                                netif_trans_update(dev->net);
                                __skb_queue_tail(&dev->txq, skb);
                        }
                }

                smp_mb();
                clear_bit(EVENT_DEV_ASLEEP, &dev->flags);
                spin_unlock_irq(&dev->txq.lock);

                if (test_bit(EVENT_DEV_OPEN, &dev->flags)) {
                        /* handle remote wakeup ASAP
                         * we cannot race against stop
                         */
                        if (netif_device_present(dev->net) &&
                                !timer_pending(&dev->delay) &&
                                !test_bit(EVENT_RX_HALT, &dev->flags))
                                        rx_alloc_submit(dev, GFP_NOIO);

                        if (!(dev->txq.qlen >= TX_QLEN(dev)))
                                netif_tx_wake_all_queues(dev->net);
                        tasklet_schedule (&dev->bh);
                }
        }

        if (test_and_clear_bit(EVENT_DEVICE_REPORT_IDLE, &dev->flags))
                usb_autopm_get_interface_no_resume(intf);

        return 0;
}
EXPORT_SYMBOL_GPL(usbnet_resume);

/*
 * Either a subdriver implements manage_power, then it is assumed to always
 * be ready to be suspended or it reports the readiness to be suspended
 * explicitly
 */
void usbnet_device_suggests_idle(struct usbnet *dev)
{
        if (!test_and_set_bit(EVENT_DEVICE_REPORT_IDLE, &dev->flags)) {
                dev->intf->needs_remote_wakeup = 1;
                usb_autopm_put_interface_async(dev->intf);
        }
}
EXPORT_SYMBOL(usbnet_device_suggests_idle);

/*
 * For devices that can do without special commands
 */
int usbnet_manage_power(struct usbnet *dev, int on)
{
        dev->intf->needs_remote_wakeup = on;
        return 0;
}
EXPORT_SYMBOL(usbnet_manage_power);

void usbnet_link_change(struct usbnet *dev, bool link, bool need_reset)
{
        /* update link after link is reseted */
        if (link && !need_reset)
                netif_carrier_on(dev->net);
        else
                netif_carrier_off(dev->net);

        if (need_reset && link)
                usbnet_defer_kevent(dev, EVENT_LINK_RESET);
        else
                usbnet_defer_kevent(dev, EVENT_LINK_CHANGE);
}
EXPORT_SYMBOL(usbnet_link_change);

/*-------------------------------------------------------------------------*/
static int __usbnet_read_cmd(struct usbnet *dev, u8 cmd, u8 reqtype,
                             u16 value, u16 index, void *data, u16 size)
{
        void *buf = NULL;
        int err = -ENOMEM;

        netdev_dbg(dev->net, "usbnet_read_cmd cmd=0x%02x reqtype=%02x"
                   " value=0x%04x index=0x%04x size=%d\n",
                   cmd, reqtype, value, index, size);

        if (size) {
                buf = kmalloc(size, GFP_NOIO);
                if (!buf)
                        goto out;
        }

        err = usb_control_msg(dev->udev, usb_rcvctrlpipe(dev->udev, 0),
                              cmd, reqtype, value, index, buf, size,
                              USB_CTRL_GET_TIMEOUT);
        if (err > 0 && err <= size) {
                if (data)
                        memcpy(data, buf, err);
                else
                        netdev_dbg(dev->net,
                                   "Huh? Data requested but thrown away.\n");
        }
        kfree(buf);
out:
        return err;
}

static int __usbnet_write_cmd(struct usbnet *dev, u8 cmd, u8 reqtype,
                              u16 value, u16 index, const void *data,
                              u16 size)
{
        void *buf = NULL;
        int err = -ENOMEM;

        netdev_dbg(dev->net, "usbnet_write_cmd cmd=0x%02x reqtype=%02x"
                   " value=0x%04x index=0x%04x size=%d\n",
                   cmd, reqtype, value, index, size);

        if (data) {
                buf = kmemdup(data, size, GFP_NOIO);
                if (!buf)
                        goto out;
        } else {
        if (size) {
            WARN_ON_ONCE(1);
            err = -EINVAL;
            goto out;
        }
    }

        err = usb_control_msg(dev->udev, usb_sndctrlpipe(dev->udev, 0),
                              cmd, reqtype, value, index, buf, size,
                              USB_CTRL_SET_TIMEOUT);
        kfree(buf);

out:
        return err;
}

/*
 * The function can't be called inside suspend/resume callback,
 * otherwise deadlock will be caused.
 */
int usbnet_read_cmd(struct usbnet *dev, u8 cmd, u8 reqtype,
                    u16 value, u16 index, void *data, u16 size)
{
        int ret;

        if (usb_autopm_get_interface(dev->intf) < 0)
                return -ENODEV;
        ret = __usbnet_read_cmd(dev, cmd, reqtype, value, index,
                                data, size);
        usb_autopm_put_interface(dev->intf);
        return ret;
}
EXPORT_SYMBOL_GPL(usbnet_read_cmd);

/*
 * The function can't be called inside suspend/resume callback,
 * otherwise deadlock will be caused.
 */
int usbnet_write_cmd(struct usbnet *dev, u8 cmd, u8 reqtype,
                     u16 value, u16 index, const void *data, u16 size)
{
        int ret;

        if (usb_autopm_get_interface(dev->intf) < 0)
                return -ENODEV;
        ret = __usbnet_write_cmd(dev, cmd, reqtype, value, index,
                                 data, size);
        usb_autopm_put_interface(dev->intf);
        return ret;
}
EXPORT_SYMBOL_GPL(usbnet_write_cmd);

/*
 * The function can be called inside suspend/resume callback safely
 * and should only be called by suspend/resume callback generally.
 */
int usbnet_read_cmd_nopm(struct usbnet *dev, u8 cmd, u8 reqtype,
                          u16 value, u16 index, void *data, u16 size)
{
        return __usbnet_read_cmd(dev, cmd, reqtype, value, index,
                                 data, size);
}
EXPORT_SYMBOL_GPL(usbnet_read_cmd_nopm);

/*
 * The function can be called inside suspend/resume callback safely
 * and should only be called by suspend/resume callback generally.
 */
int usbnet_write_cmd_nopm(struct usbnet *dev, u8 cmd, u8 reqtype,
                          u16 value, u16 index, const void *data,
                          u16 size)
{
        return __usbnet_write_cmd(dev, cmd, reqtype, value, index,
                                  data, size);
}
EXPORT_SYMBOL_GPL(usbnet_write_cmd_nopm);

static void usbnet_async_cmd_cb(struct urb *urb)
{
        struct usb_ctrlrequest *req = (struct usb_ctrlrequest *)urb->context;
        int status = urb->status;

        if (status < 0)
                dev_dbg(&urb->dev->dev, "%s failed with %d",
                        __func__, status);

        kfree(req);
        usb_free_urb(urb);
}

/*
 * The caller must make sure that device can't be put into suspend
 * state until the control URB completes.
 */
int usbnet_write_cmd_async(struct usbnet *dev, u8 cmd, u8 reqtype,
                           u16 value, u16 index, const void *data, u16 size)
{
        struct usb_ctrlrequest *req;
        struct urb *urb;
        int err = -ENOMEM;
        void *buf = NULL;

        netdev_dbg(dev->net, "usbnet_write_cmd cmd=0x%02x reqtype=%02x"
                   " value=0x%04x index=0x%04x size=%d\n",
                   cmd, reqtype, value, index, size);

        urb = usb_alloc_urb(0, GFP_ATOMIC);
        if (!urb)
                goto fail;

        if (data) {
                buf = kmemdup(data, size, GFP_ATOMIC);
                if (!buf) {
                        netdev_err(dev->net, "Error allocating buffer"
                                   " in %s!\n", __func__);
                        goto fail_free_urb;
                }
        }

        req = kmalloc(sizeof(struct usb_ctrlrequest), GFP_ATOMIC);
        if (!req)
                goto fail_free_buf;

        req->bRequestType = reqtype;
        req->bRequest = cmd;
        req->wValue = cpu_to_le16(value);
        req->wIndex = cpu_to_le16(index);
        req->wLength = cpu_to_le16(size);

        usb_fill_control_urb(urb, dev->udev,
                             usb_sndctrlpipe(dev->udev, 0),
                             (void *)req, buf, size,
                             usbnet_async_cmd_cb, req);
        urb->transfer_flags |= URB_FREE_BUFFER;

        err = usb_submit_urb(urb, GFP_ATOMIC);
        if (err < 0) {
                netdev_err(dev->net, "Error submitting the control"
                           " message: status=%d\n", err);
                goto fail_free_all;
        }
        return 0;

fail_free_all:
        kfree(req);
fail_free_buf:
        kfree(buf);
        /*
         * avoid a double free
         * needed because the flag can be set only
         * after filling the URB
         */
        urb->transfer_flags = 0;
fail_free_urb:
        usb_free_urb(urb);
fail:
        return err;

}
EXPORT_SYMBOL_GPL(usbnet_write_cmd_async);
/*-------------------------------------------------------------------------*/

static int __init usbnet_init(void)
{
        /* Compiler should optimize this out. */
        BUILD_BUG_ON(
                sizeof_field(struct sk_buff, cb) < sizeof(struct skb_data));

        return 0;
}
module_init(usbnet_init);

static void __exit usbnet_exit(void)
{
}
module_exit(usbnet_exit);

MODULE_AUTHOR("David Brownell");
MODULE_DESCRIPTION("USB network driver framework");
MODULE_LICENSE("GPL");


























    1 









    1 








































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
// SPDX-License-Identifier: LGPL-2.1
/*
 *   SPNEGO upcall management for CIFS
 *
 *   Copyright (c) 2007 Red Hat, Inc.
 *   Author(s): Jeff Layton (jlayton@redhat.com)
 *
 */

#include <linux/list.h>
#include <linux/slab.h>
#include <linux/string.h>
#include <keys/user-type.h>
#include <linux/key-type.h>
#include <linux/keyctl.h>
#include <linux/inet.h>
#include "cifsglob.h"
#include "cifs_spnego.h"
#include "cifs_debug.h"
#include "cifsproto.h"
static const struct cred *spnego_cred;

/* create a new cifs key */
static int
cifs_spnego_key_instantiate(struct key *key, struct key_preparsed_payload *prep)
{
        char *payload;
        int ret;

        ret = -ENOMEM;
        payload = kmemdup(prep->data, prep->datalen, GFP_KERNEL);
        if (!payload)
                goto error;

        /* attach the data */
        key->payload.data[0] = payload;
        ret = 0;

error:
        return ret;
}

static void
cifs_spnego_key_destroy(struct key *key)
{
        kfree(key->payload.data[0]);
}


/*
 * keytype for CIFS spnego keys
 */
struct key_type cifs_spnego_key_type = {
        .name                = "cifs.spnego",
        .instantiate        = cifs_spnego_key_instantiate,
        .destroy        = cifs_spnego_key_destroy,
        .describe        = user_describe,
};

/* length of longest version string e.g.  strlen("ver=0xFF") */
#define MAX_VER_STR_LEN                8

/* length of longest security mechanism name, eg in future could have
 * strlen(";sec=ntlmsspi") */
#define MAX_MECH_STR_LEN        13

/* strlen of ";host=" */
#define HOST_KEY_LEN                6

/* strlen of ";ip4=" or ";ip6=" */
#define IP_KEY_LEN                5

/* strlen of ";uid=0x" */
#define UID_KEY_LEN                7

/* strlen of ";creduid=0x" */
#define CREDUID_KEY_LEN                11

/* strlen of ";user=" */
#define USER_KEY_LEN                6

/* strlen of ";pid=0x" */
#define PID_KEY_LEN                7

/* get a key struct with a SPNEGO security blob, suitable for session setup */
struct key *
cifs_get_spnego_key(struct cifs_ses *sesInfo,
                    struct TCP_Server_Info *server)
{
        struct sockaddr_in *sa = (struct sockaddr_in *) &server->dstaddr;
        struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *) &server->dstaddr;
        char *description, *dp;
        size_t desc_len;
        struct key *spnego_key;
        const char *hostname = server->hostname;
        const struct cred *saved_cred;

        /* length of fields (with semicolons): ver=0xyz ip4=ipaddress
           host=hostname sec=mechanism uid=0xFF user=username */
        desc_len = MAX_VER_STR_LEN +
                   HOST_KEY_LEN + strlen(hostname) +
                   IP_KEY_LEN + INET6_ADDRSTRLEN +
                   MAX_MECH_STR_LEN +
                   UID_KEY_LEN + (sizeof(uid_t) * 2) +
                   CREDUID_KEY_LEN + (sizeof(uid_t) * 2) +
                   PID_KEY_LEN + (sizeof(pid_t) * 2) + 1;

        if (sesInfo->user_name)
                desc_len += USER_KEY_LEN + strlen(sesInfo->user_name);

        spnego_key = ERR_PTR(-ENOMEM);
        description = kzalloc(desc_len, GFP_KERNEL);
        if (description == NULL)
                goto out;

        dp = description;
        /* start with version and hostname portion of UNC string */
        spnego_key = ERR_PTR(-EINVAL);
        sprintf(dp, "ver=0x%x;host=%s;", CIFS_SPNEGO_UPCALL_VERSION,
                hostname);
        dp = description + strlen(description);

        /* add the server address */
        if (server->dstaddr.ss_family == AF_INET)
                sprintf(dp, "ip4=%pI4", &sa->sin_addr);
        else if (server->dstaddr.ss_family == AF_INET6)
                sprintf(dp, "ip6=%pI6", &sa6->sin6_addr);
        else
                goto out;

        dp = description + strlen(description);

        /* for now, only sec=krb5 and sec=mskrb5 are valid */
        if (server->sec_kerberos)
                sprintf(dp, ";sec=krb5");
        else if (server->sec_mskerberos)
                sprintf(dp, ";sec=mskrb5");
        else {
                cifs_dbg(VFS, "unknown or missing server auth type, use krb5\n");
                sprintf(dp, ";sec=krb5");
        }

        dp = description + strlen(description);
        sprintf(dp, ";uid=0x%x",
                from_kuid_munged(&init_user_ns, sesInfo->linux_uid));

        dp = description + strlen(description);
        sprintf(dp, ";creduid=0x%x",
                from_kuid_munged(&init_user_ns, sesInfo->cred_uid));

        if (sesInfo->user_name) {
                dp = description + strlen(description);
                sprintf(dp, ";user=%s", sesInfo->user_name);
        }

        dp = description + strlen(description);
        sprintf(dp, ";pid=0x%x", current->pid);

        cifs_dbg(FYI, "key description = %s\n", description);
        saved_cred = override_creds(spnego_cred);
        spnego_key = request_key(&cifs_spnego_key_type, description, "");
        revert_creds(saved_cred);

#ifdef CONFIG_CIFS_DEBUG2
        if (cifsFYI && !IS_ERR(spnego_key)) {
                struct cifs_spnego_msg *msg = spnego_key->payload.data[0];
                cifs_dump_mem("SPNEGO reply blob:", msg->data, min(1024U,
                                msg->secblob_len + msg->sesskey_len));
        }
#endif /* CONFIG_CIFS_DEBUG2 */

out:
        kfree(description);
        return spnego_key;
}

int
init_cifs_spnego(void)
{
        struct cred *cred;
        struct key *keyring;
        int ret;

        cifs_dbg(FYI, "Registering the %s key type\n",
                 cifs_spnego_key_type.name);

        /*
         * Create an override credential set with special thread keyring for
         * spnego upcalls.
         */

        cred = prepare_kernel_cred(&init_task);
        if (!cred)
                return -ENOMEM;

        keyring = keyring_alloc(".cifs_spnego",
                                GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, cred,
                                (KEY_POS_ALL & ~KEY_POS_SETATTR) |
                                KEY_USR_VIEW | KEY_USR_READ,
                                KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL);
        if (IS_ERR(keyring)) {
                ret = PTR_ERR(keyring);
                goto failed_put_cred;
        }

        ret = register_key_type(&cifs_spnego_key_type);
        if (ret < 0)
                goto failed_put_key;

        /*
         * instruct request_key() to use this special keyring as a cache for
         * the results it looks up
         */
        set_bit(KEY_FLAG_ROOT_CAN_CLEAR, &keyring->flags);
        cred->thread_keyring = keyring;
        cred->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING;
        spnego_cred = cred;

        cifs_dbg(FYI, "cifs spnego keyring: %d\n", key_serial(keyring));
        return 0;

failed_put_key:
        key_put(keyring);
failed_put_cred:
        put_cred(cred);
        return ret;
}

void
exit_cifs_spnego(void)
{
        key_revoke(spnego_cred->thread_keyring);
        unregister_key_type(&cifs_spnego_key_type);
        put_cred(spnego_cred);
        cifs_dbg(FYI, "Unregistered %s key type\n", cifs_spnego_key_type.name);
}


















  225 












1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PATH_H
#define _LINUX_PATH_H

struct dentry;
struct vfsmount;

struct path {
        struct vfsmount *mnt;
        struct dentry *dentry;
} __randomize_layout;

extern void path_get(const struct path *);
extern void path_put(const struct path *);

static inline int path_equal(const struct path *path1, const struct path *path2)
{
        return path1->mnt == path2->mnt && path1->dentry == path2->dentry;
}

/*
 * Cleanup macro for use with __free(path_put). Avoids dereference and
 * copying @path unlike DEFINE_FREE(). path_put() will handle the empty
 * path correctly just ensure @path is initialized:
 *
 * struct path path __free(path_put) = {};
 */
#define __free_path_put path_put

#endif  /* _LINUX_PATH_H */






















   39 









































   40 









    3 





   39 






  141 



    3 













  127 
    5 






    2 





   42 











   30 
   27 


   17 
    1 












   65 







   29 





   12 




    2 













   22 

    6 







   23 


   23 






    7 










    7 














   63 











   19 





   10 






    1 

    1 


    1 





    8 




   27 









    1 







   27 





   27 




   27 









   85 









    7 





   23 






   43 
   54 






   53 






   70 
    7 
    5 







   16 

   74 

   22 














   59 
   36 






   74 
    7 
   10 









   73 







   13 


    3 
    2 

















   48 
















    2 

    3 

   44 






1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2012-2013 Samsung Electronics Co., Ltd.
 */

#include <linux/slab.h>
#include <asm/unaligned.h>
#include <linux/buffer_head.h>
#include <linux/blkdev.h>

#include "exfat_raw.h"
#include "exfat_fs.h"

static int exfat_mirror_bh(struct super_block *sb, sector_t sec,
                struct buffer_head *bh)
{
        struct buffer_head *c_bh;
        struct exfat_sb_info *sbi = EXFAT_SB(sb);
        sector_t sec2;
        int err = 0;

        if (sbi->FAT2_start_sector != sbi->FAT1_start_sector) {
                sec2 = sec - sbi->FAT1_start_sector + sbi->FAT2_start_sector;
                c_bh = sb_getblk(sb, sec2);
                if (!c_bh)
                        return -ENOMEM;
                memcpy(c_bh->b_data, bh->b_data, sb->s_blocksize);
                set_buffer_uptodate(c_bh);
                mark_buffer_dirty(c_bh);
                if (sb->s_flags & SB_SYNCHRONOUS)
                        err = sync_dirty_buffer(c_bh);
                brelse(c_bh);
        }

        return err;
}

static int __exfat_ent_get(struct super_block *sb, unsigned int loc,
                unsigned int *content)
{
        unsigned int off;
        sector_t sec;
        struct buffer_head *bh;

        sec = FAT_ENT_OFFSET_SECTOR(sb, loc);
        off = FAT_ENT_OFFSET_BYTE_IN_SECTOR(sb, loc);

        bh = sb_bread(sb, sec);
        if (!bh)
                return -EIO;

        *content = le32_to_cpu(*(__le32 *)(&bh->b_data[off]));

        /* remap reserved clusters to simplify code */
        if (*content > EXFAT_BAD_CLUSTER)
                *content = EXFAT_EOF_CLUSTER;

        brelse(bh);
        return 0;
}

int exfat_ent_set(struct super_block *sb, unsigned int loc,
                unsigned int content)
{
        unsigned int off;
        sector_t sec;
        __le32 *fat_entry;
        struct buffer_head *bh;

        sec = FAT_ENT_OFFSET_SECTOR(sb, loc);
        off = FAT_ENT_OFFSET_BYTE_IN_SECTOR(sb, loc);

        bh = sb_bread(sb, sec);
        if (!bh)
                return -EIO;

        fat_entry = (__le32 *)&(bh->b_data[off]);
        *fat_entry = cpu_to_le32(content);
        exfat_update_bh(bh, sb->s_flags & SB_SYNCHRONOUS);
        exfat_mirror_bh(sb, sec, bh);
        brelse(bh);
        return 0;
}

int exfat_ent_get(struct super_block *sb, unsigned int loc,
                unsigned int *content)
{
        struct exfat_sb_info *sbi = EXFAT_SB(sb);
        int err;

        if (!is_valid_cluster(sbi, loc)) {
                exfat_fs_error(sb, "invalid access to FAT (entry 0x%08x)",
                        loc);
                return -EIO;
        }

        err = __exfat_ent_get(sb, loc, content);
        if (err) {
                exfat_fs_error(sb,
                        "failed to access to FAT (entry 0x%08x, err:%d)",
                        loc, err);
                return err;
        }

        if (*content == EXFAT_FREE_CLUSTER) {
                exfat_fs_error(sb,
                        "invalid access to FAT free cluster (entry 0x%08x)",
                        loc);
                return -EIO;
        }

        if (*content == EXFAT_BAD_CLUSTER) {
                exfat_fs_error(sb,
                        "invalid access to FAT bad cluster (entry 0x%08x)",
                        loc);
                return -EIO;
        }

        if (*content != EXFAT_EOF_CLUSTER && !is_valid_cluster(sbi, *content)) {
                exfat_fs_error(sb,
                        "invalid access to FAT (entry 0x%08x) bogus content (0x%08x)",
                        loc, *content);
                return -EIO;
        }

        return 0;
}

int exfat_chain_cont_cluster(struct super_block *sb, unsigned int chain,
                unsigned int len)
{
        if (!len)
                return 0;

        while (len > 1) {
                if (exfat_ent_set(sb, chain, chain + 1))
                        return -EIO;
                chain++;
                len--;
        }

        if (exfat_ent_set(sb, chain, EXFAT_EOF_CLUSTER))
                return -EIO;
        return 0;
}

/* This function must be called with bitmap_lock held */
static int __exfat_free_cluster(struct inode *inode, struct exfat_chain *p_chain)
{
        struct super_block *sb = inode->i_sb;
        struct exfat_sb_info *sbi = EXFAT_SB(sb);
        int cur_cmap_i, next_cmap_i;
        unsigned int num_clusters = 0;
        unsigned int clu;

        /* invalid cluster number */
        if (p_chain->dir == EXFAT_FREE_CLUSTER ||
            p_chain->dir == EXFAT_EOF_CLUSTER ||
            p_chain->dir < EXFAT_FIRST_CLUSTER)
                return 0;

        /* no cluster to truncate */
        if (p_chain->size == 0)
                return 0;

        /* check cluster validation */
        if (!is_valid_cluster(sbi, p_chain->dir)) {
                exfat_err(sb, "invalid start cluster (%u)", p_chain->dir);
                return -EIO;
        }

        clu = p_chain->dir;

        cur_cmap_i = next_cmap_i =
                BITMAP_OFFSET_SECTOR_INDEX(sb, CLUSTER_TO_BITMAP_ENT(clu));

        if (p_chain->flags == ALLOC_NO_FAT_CHAIN) {
                unsigned int last_cluster = p_chain->dir + p_chain->size - 1;
                do {
                        bool sync = false;

                        if (clu < last_cluster)
                                next_cmap_i =
                                  BITMAP_OFFSET_SECTOR_INDEX(sb, CLUSTER_TO_BITMAP_ENT(clu+1));

                        /* flush bitmap only if index would be changed or for last cluster */
                        if (clu == last_cluster || cur_cmap_i != next_cmap_i) {
                                sync = true;
                                cur_cmap_i = next_cmap_i;
                        }

                        exfat_clear_bitmap(inode, clu, (sync && IS_DIRSYNC(inode)));
                        clu++;
                        num_clusters++;
                } while (num_clusters < p_chain->size);
        } else {
                do {
                        bool sync = false;
                        unsigned int n_clu = clu;
                        int err = exfat_get_next_cluster(sb, &n_clu);

                        if (err || n_clu == EXFAT_EOF_CLUSTER)
                                sync = true;
                        else
                                next_cmap_i =
                                  BITMAP_OFFSET_SECTOR_INDEX(sb, CLUSTER_TO_BITMAP_ENT(n_clu));

                        if (cur_cmap_i != next_cmap_i) {
                                sync = true;
                                cur_cmap_i = next_cmap_i;
                        }

                        exfat_clear_bitmap(inode, clu, (sync && IS_DIRSYNC(inode)));
                        clu = n_clu;
                        num_clusters++;

                        if (err)
                                goto dec_used_clus;
                } while (clu != EXFAT_EOF_CLUSTER);
        }

dec_used_clus:
        sbi->used_clusters -= num_clusters;
        return 0;
}

int exfat_free_cluster(struct inode *inode, struct exfat_chain *p_chain)
{
        int ret = 0;

        mutex_lock(&EXFAT_SB(inode->i_sb)->bitmap_lock);
        ret = __exfat_free_cluster(inode, p_chain);
        mutex_unlock(&EXFAT_SB(inode->i_sb)->bitmap_lock);

        return ret;
}

int exfat_find_last_cluster(struct super_block *sb, struct exfat_chain *p_chain,
                unsigned int *ret_clu)
{
        unsigned int clu, next;
        unsigned int count = 0;

        next = p_chain->dir;
        if (p_chain->flags == ALLOC_NO_FAT_CHAIN) {
                *ret_clu = next + p_chain->size - 1;
                return 0;
        }

        do {
                count++;
                clu = next;
                if (exfat_ent_get(sb, clu, &next))
                        return -EIO;
        } while (next != EXFAT_EOF_CLUSTER);

        if (p_chain->size != count) {
                exfat_fs_error(sb,
                        "bogus directory size (clus : ondisk(%d) != counted(%d))",
                        p_chain->size, count);
                return -EIO;
        }

        *ret_clu = clu;
        return 0;
}

int exfat_zeroed_cluster(struct inode *dir, unsigned int clu)
{
        struct super_block *sb = dir->i_sb;
        struct exfat_sb_info *sbi = EXFAT_SB(sb);
        struct buffer_head *bh;
        sector_t blknr, last_blknr, i;

        blknr = exfat_cluster_to_sector(sbi, clu);
        last_blknr = blknr + sbi->sect_per_clus;

        if (last_blknr > sbi->num_sectors && sbi->num_sectors > 0) {
                exfat_fs_error_ratelimit(sb,
                        "%s: out of range(sect:%llu len:%u)",
                        __func__, (unsigned long long)blknr,
                        sbi->sect_per_clus);
                return -EIO;
        }

        /* Zeroing the unused blocks on this cluster */
        for (i = blknr; i < last_blknr; i++) {
                bh = sb_getblk(sb, i);
                if (!bh)
                        return -ENOMEM;

                memset(bh->b_data, 0, sb->s_blocksize);
                set_buffer_uptodate(bh);
                mark_buffer_dirty(bh);
                brelse(bh);
        }

        if (IS_DIRSYNC(dir))
                return sync_blockdev_range(sb->s_bdev,
                                EXFAT_BLK_TO_B(blknr, sb),
                                EXFAT_BLK_TO_B(last_blknr, sb) - 1);

        return 0;
}

int exfat_alloc_cluster(struct inode *inode, unsigned int num_alloc,
                struct exfat_chain *p_chain, bool sync_bmap)
{
        int ret = -ENOSPC;
        unsigned int total_cnt;
        unsigned int hint_clu, new_clu, last_clu = EXFAT_EOF_CLUSTER;
        struct super_block *sb = inode->i_sb;
        struct exfat_sb_info *sbi = EXFAT_SB(sb);

        total_cnt = EXFAT_DATA_CLUSTER_COUNT(sbi);

        if (unlikely(total_cnt < sbi->used_clusters)) {
                exfat_fs_error_ratelimit(sb,
                        "%s: invalid used clusters(t:%u,u:%u)\n",
                        __func__, total_cnt, sbi->used_clusters);
                return -EIO;
        }

        if (num_alloc > total_cnt - sbi->used_clusters)
                return -ENOSPC;

        mutex_lock(&sbi->bitmap_lock);

        hint_clu = p_chain->dir;
        /* find new cluster */
        if (hint_clu == EXFAT_EOF_CLUSTER) {
                if (sbi->clu_srch_ptr < EXFAT_FIRST_CLUSTER) {
                        exfat_err(sb, "sbi->clu_srch_ptr is invalid (%u)",
                                  sbi->clu_srch_ptr);
                        sbi->clu_srch_ptr = EXFAT_FIRST_CLUSTER;
                }

                hint_clu = exfat_find_free_bitmap(sb, sbi->clu_srch_ptr);
                if (hint_clu == EXFAT_EOF_CLUSTER) {
                        ret = -ENOSPC;
                        goto unlock;
                }
        }

        /* check cluster validation */
        if (!is_valid_cluster(sbi, hint_clu)) {
                if (hint_clu != sbi->num_clusters)
                        exfat_err(sb, "hint_cluster is invalid (%u), rewind to the first cluster",
                                        hint_clu);
                hint_clu = EXFAT_FIRST_CLUSTER;
                p_chain->flags = ALLOC_FAT_CHAIN;
        }

        p_chain->dir = EXFAT_EOF_CLUSTER;

        while ((new_clu = exfat_find_free_bitmap(sb, hint_clu)) !=
               EXFAT_EOF_CLUSTER) {
                if (new_clu != hint_clu &&
                    p_chain->flags == ALLOC_NO_FAT_CHAIN) {
                        if (exfat_chain_cont_cluster(sb, p_chain->dir,
                                        p_chain->size)) {
                                ret = -EIO;
                                goto free_cluster;
                        }
                        p_chain->flags = ALLOC_FAT_CHAIN;
                }

                /* update allocation bitmap */
                if (exfat_set_bitmap(inode, new_clu, sync_bmap)) {
                        ret = -EIO;
                        goto free_cluster;
                }

                /* update FAT table */
                if (p_chain->flags == ALLOC_FAT_CHAIN) {
                        if (exfat_ent_set(sb, new_clu, EXFAT_EOF_CLUSTER)) {
                                ret = -EIO;
                                goto free_cluster;
                        }
                }

                if (p_chain->dir == EXFAT_EOF_CLUSTER) {
                        p_chain->dir = new_clu;
                } else if (p_chain->flags == ALLOC_FAT_CHAIN) {
                        if (exfat_ent_set(sb, last_clu, new_clu)) {
                                ret = -EIO;
                                goto free_cluster;
                        }
                }
                p_chain->size++;

                last_clu = new_clu;

                if (p_chain->size == num_alloc) {
                        sbi->clu_srch_ptr = hint_clu;
                        sbi->used_clusters += num_alloc;

                        mutex_unlock(&sbi->bitmap_lock);
                        return 0;
                }

                hint_clu = new_clu + 1;
                if (hint_clu >= sbi->num_clusters) {
                        hint_clu = EXFAT_FIRST_CLUSTER;

                        if (p_chain->flags == ALLOC_NO_FAT_CHAIN) {
                                if (exfat_chain_cont_cluster(sb, p_chain->dir,
                                                p_chain->size)) {
                                        ret = -EIO;
                                        goto free_cluster;
                                }
                                p_chain->flags = ALLOC_FAT_CHAIN;
                        }
                }
        }
free_cluster:
        __exfat_free_cluster(inode, p_chain);
unlock:
        mutex_unlock(&sbi->bitmap_lock);
        return ret;
}

int exfat_count_num_clusters(struct super_block *sb,
                struct exfat_chain *p_chain, unsigned int *ret_count)
{
        unsigned int i, count;
        unsigned int clu;
        struct exfat_sb_info *sbi = EXFAT_SB(sb);

        if (!p_chain->dir || p_chain->dir == EXFAT_EOF_CLUSTER) {
                *ret_count = 0;
                return 0;
        }

        if (p_chain->flags == ALLOC_NO_FAT_CHAIN) {
                *ret_count = p_chain->size;
                return 0;
        }

        clu = p_chain->dir;
        count = 0;
        for (i = EXFAT_FIRST_CLUSTER; i < sbi->num_clusters; i++) {
                count++;
                if (exfat_ent_get(sb, clu, &clu))
                        return -EIO;
                if (clu == EXFAT_EOF_CLUSTER)
                        break;
        }

        *ret_count = count;
        return 0;
}


























    2 











    2 












   31 



1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
/*
 *  linux/drivers/video/fb_notify.c
 *
 *  Copyright (C) 2006 Antonino Daplas <adaplas@pol.net>
 *
 *        2001 - Documented with DocBook
 *        - Brad Douglas <brad@neruo.com>
 *
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file COPYING in the main directory of this archive
 * for more details.
 */
#include <linux/fb.h>
#include <linux/notifier.h>
#include <linux/export.h>

static BLOCKING_NOTIFIER_HEAD(fb_notifier_list);

/**
 *        fb_register_client - register a client notifier
 *        @nb: notifier block to callback on events
 *
 *        Return: 0 on success, negative error code on failure.
 */
int fb_register_client(struct notifier_block *nb)
{
        return blocking_notifier_chain_register(&fb_notifier_list, nb);
}
EXPORT_SYMBOL(fb_register_client);

/**
 *        fb_unregister_client - unregister a client notifier
 *        @nb: notifier block to callback on events
 *
 *        Return: 0 on success, negative error code on failure.
 */
int fb_unregister_client(struct notifier_block *nb)
{
        return blocking_notifier_chain_unregister(&fb_notifier_list, nb);
}
EXPORT_SYMBOL(fb_unregister_client);

/**
 * fb_notifier_call_chain - notify clients of fb_events
 * @val: value passed to callback
 * @v: pointer passed to callback
 *
 * Return: The return value of the last notifier function
 */
int fb_notifier_call_chain(unsigned long val, void *v)
{
        return blocking_notifier_call_chain(&fb_notifier_list, val, v);
}
EXPORT_SYMBOL_GPL(fb_notifier_call_chain);






























































    4 

    1 

    3 







































    4 

    1 

    3 




    3 









    2 









    2 





    2 








    2 
















    1 
    6 

























































































    1 








    6 


    7 




    1 


    1 





    1 
    3 












   10 









    6 

    5 







   10 















    5 







    4 





    5 






    5 





    5 
    5 



    5 















    1 






    4 







    5 














    7 
    3 
    1 

    2 









    7 




    7 



    2 




   11 


   10 
   10 




    8 










    8 





    8 
    8 



    5 












   22 

















    1 





    1 


    1 








    1 






    7 








    8 
    3 

























    1 










    2 




    8 
    3 
    2 








    2 

    3 


    2 

    3 







    8 








    5 



   13 
    1 




   13 
    1 
   13 

   13 














    6 



















    6 



    4 
    2 



    4 
    2 






    6 






















































































































































































































   41 


   42 






























1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * net/sched/ife.c        Inter-FE action based on ForCES WG InterFE LFB
 *
 *                Refer to:
 *                draft-ietf-forces-interfelfb-03
 *                and
 *                netdev01 paper:
 *                "Distributing Linux Traffic Control Classifier-Action
 *                Subsystem"
 *                Authors: Jamal Hadi Salim and Damascene M. Joachimpillai
 *
 * copyright Jamal Hadi Salim (2015)
*/

#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/errno.h>
#include <linux/skbuff.h>
#include <linux/rtnetlink.h>
#include <linux/module.h>
#include <linux/init.h>
#include <net/net_namespace.h>
#include <net/netlink.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <uapi/linux/tc_act/tc_ife.h>
#include <net/tc_act/tc_ife.h>
#include <linux/etherdevice.h>
#include <net/ife.h>
#include <net/tc_wrapper.h>

static int max_metacnt = IFE_META_MAX + 1;
static struct tc_action_ops act_ife_ops;

static const struct nla_policy ife_policy[TCA_IFE_MAX + 1] = {
        [TCA_IFE_PARMS] = { .len = sizeof(struct tc_ife)},
        [TCA_IFE_DMAC] = { .len = ETH_ALEN},
        [TCA_IFE_SMAC] = { .len = ETH_ALEN},
        [TCA_IFE_TYPE] = { .type = NLA_U16},
};

int ife_encode_meta_u16(u16 metaval, void *skbdata, struct tcf_meta_info *mi)
{
        u16 edata = 0;

        if (mi->metaval)
                edata = *(u16 *)mi->metaval;
        else if (metaval)
                edata = metaval;

        if (!edata) /* will not encode */
                return 0;

        edata = htons(edata);
        return ife_tlv_meta_encode(skbdata, mi->metaid, 2, &edata);
}
EXPORT_SYMBOL_GPL(ife_encode_meta_u16);

int ife_get_meta_u32(struct sk_buff *skb, struct tcf_meta_info *mi)
{
        if (mi->metaval)
                return nla_put_u32(skb, mi->metaid, *(u32 *)mi->metaval);
        else
                return nla_put(skb, mi->metaid, 0, NULL);
}
EXPORT_SYMBOL_GPL(ife_get_meta_u32);

int ife_check_meta_u32(u32 metaval, struct tcf_meta_info *mi)
{
        if (metaval || mi->metaval)
                return 8; /* T+L+V == 2+2+4 */

        return 0;
}
EXPORT_SYMBOL_GPL(ife_check_meta_u32);

int ife_check_meta_u16(u16 metaval, struct tcf_meta_info *mi)
{
        if (metaval || mi->metaval)
                return 8; /* T+L+(V) == 2+2+(2+2bytepad) */

        return 0;
}
EXPORT_SYMBOL_GPL(ife_check_meta_u16);

int ife_encode_meta_u32(u32 metaval, void *skbdata, struct tcf_meta_info *mi)
{
        u32 edata = metaval;

        if (mi->metaval)
                edata = *(u32 *)mi->metaval;
        else if (metaval)
                edata = metaval;

        if (!edata) /* will not encode */
                return 0;

        edata = htonl(edata);
        return ife_tlv_meta_encode(skbdata, mi->metaid, 4, &edata);
}
EXPORT_SYMBOL_GPL(ife_encode_meta_u32);

int ife_get_meta_u16(struct sk_buff *skb, struct tcf_meta_info *mi)
{
        if (mi->metaval)
                return nla_put_u16(skb, mi->metaid, *(u16 *)mi->metaval);
        else
                return nla_put(skb, mi->metaid, 0, NULL);
}
EXPORT_SYMBOL_GPL(ife_get_meta_u16);

int ife_alloc_meta_u32(struct tcf_meta_info *mi, void *metaval, gfp_t gfp)
{
        mi->metaval = kmemdup(metaval, sizeof(u32), gfp);
        if (!mi->metaval)
                return -ENOMEM;

        return 0;
}
EXPORT_SYMBOL_GPL(ife_alloc_meta_u32);

int ife_alloc_meta_u16(struct tcf_meta_info *mi, void *metaval, gfp_t gfp)
{
        mi->metaval = kmemdup(metaval, sizeof(u16), gfp);
        if (!mi->metaval)
                return -ENOMEM;

        return 0;
}
EXPORT_SYMBOL_GPL(ife_alloc_meta_u16);

void ife_release_meta_gen(struct tcf_meta_info *mi)
{
        kfree(mi->metaval);
}
EXPORT_SYMBOL_GPL(ife_release_meta_gen);

int ife_validate_meta_u32(void *val, int len)
{
        if (len == sizeof(u32))
                return 0;

        return -EINVAL;
}
EXPORT_SYMBOL_GPL(ife_validate_meta_u32);

int ife_validate_meta_u16(void *val, int len)
{
        /* length will not include padding */
        if (len == sizeof(u16))
                return 0;

        return -EINVAL;
}
EXPORT_SYMBOL_GPL(ife_validate_meta_u16);

static LIST_HEAD(ifeoplist);
static DEFINE_RWLOCK(ife_mod_lock);

static struct tcf_meta_ops *find_ife_oplist(u16 metaid)
{
        struct tcf_meta_ops *o;

        read_lock(&ife_mod_lock);
        list_for_each_entry(o, &ifeoplist, list) {
                if (o->metaid == metaid) {
                        if (!try_module_get(o->owner))
                                o = NULL;
                        read_unlock(&ife_mod_lock);
                        return o;
                }
        }
        read_unlock(&ife_mod_lock);

        return NULL;
}

int register_ife_op(struct tcf_meta_ops *mops)
{
        struct tcf_meta_ops *m;

        if (!mops->metaid || !mops->metatype || !mops->name ||
            !mops->check_presence || !mops->encode || !mops->decode ||
            !mops->get || !mops->alloc)
                return -EINVAL;

        write_lock(&ife_mod_lock);

        list_for_each_entry(m, &ifeoplist, list) {
                if (m->metaid == mops->metaid ||
                    (strcmp(mops->name, m->name) == 0)) {
                        write_unlock(&ife_mod_lock);
                        return -EEXIST;
                }
        }

        if (!mops->release)
                mops->release = ife_release_meta_gen;

        list_add_tail(&mops->list, &ifeoplist);
        write_unlock(&ife_mod_lock);
        return 0;
}
EXPORT_SYMBOL_GPL(unregister_ife_op);

int unregister_ife_op(struct tcf_meta_ops *mops)
{
        struct tcf_meta_ops *m;
        int err = -ENOENT;

        write_lock(&ife_mod_lock);
        list_for_each_entry(m, &ifeoplist, list) {
                if (m->metaid == mops->metaid) {
                        list_del(&mops->list);
                        err = 0;
                        break;
                }
        }
        write_unlock(&ife_mod_lock);

        return err;
}
EXPORT_SYMBOL_GPL(register_ife_op);

static int ife_validate_metatype(struct tcf_meta_ops *ops, void *val, int len)
{
        int ret = 0;
        /* XXX: unfortunately cant use nla_policy at this point
        * because a length of 0 is valid in the case of
        * "allow". "use" semantics do enforce for proper
        * length and i couldve use nla_policy but it makes it hard
        * to use it just for that..
        */
        if (ops->validate)
                return ops->validate(val, len);

        if (ops->metatype == NLA_U32)
                ret = ife_validate_meta_u32(val, len);
        else if (ops->metatype == NLA_U16)
                ret = ife_validate_meta_u16(val, len);

        return ret;
}

#ifdef CONFIG_MODULES
static const char *ife_meta_id2name(u32 metaid)
{
        switch (metaid) {
        case IFE_META_SKBMARK:
                return "skbmark";
        case IFE_META_PRIO:
                return "skbprio";
        case IFE_META_TCINDEX:
                return "tcindex";
        default:
                return "unknown";
        }
}
#endif

/* called when adding new meta information
*/
static int load_metaops_and_vet(u32 metaid, void *val, int len, bool rtnl_held)
{
        struct tcf_meta_ops *ops = find_ife_oplist(metaid);
        int ret = 0;

        if (!ops) {
                ret = -ENOENT;
#ifdef CONFIG_MODULES
                if (rtnl_held)
                        rtnl_unlock();
                request_module("ife-meta-%s", ife_meta_id2name(metaid));
                if (rtnl_held)
                        rtnl_lock();
                ops = find_ife_oplist(metaid);
#endif
        }

        if (ops) {
                ret = 0;
                if (len)
                        ret = ife_validate_metatype(ops, val, len);

                module_put(ops->owner);
        }

        return ret;
}

/* called when adding new meta information
*/
static int __add_metainfo(const struct tcf_meta_ops *ops,
                          struct tcf_ife_info *ife, u32 metaid, void *metaval,
                          int len, bool atomic, bool exists)
{
        struct tcf_meta_info *mi = NULL;
        int ret = 0;

        mi = kzalloc(sizeof(*mi), atomic ? GFP_ATOMIC : GFP_KERNEL);
        if (!mi)
                return -ENOMEM;

        mi->metaid = metaid;
        mi->ops = ops;
        if (len > 0) {
                ret = ops->alloc(mi, metaval, atomic ? GFP_ATOMIC : GFP_KERNEL);
                if (ret != 0) {
                        kfree(mi);
                        return ret;
                }
        }

        if (exists)
                spin_lock_bh(&ife->tcf_lock);
        list_add_tail(&mi->metalist, &ife->metalist);
        if (exists)
                spin_unlock_bh(&ife->tcf_lock);

        return ret;
}

static int add_metainfo_and_get_ops(const struct tcf_meta_ops *ops,
                                    struct tcf_ife_info *ife, u32 metaid,
                                    bool exists)
{
        int ret;

        if (!try_module_get(ops->owner))
                return -ENOENT;
        ret = __add_metainfo(ops, ife, metaid, NULL, 0, true, exists);
        if (ret)
                module_put(ops->owner);
        return ret;
}

static int add_metainfo(struct tcf_ife_info *ife, u32 metaid, void *metaval,
                        int len, bool exists)
{
        const struct tcf_meta_ops *ops = find_ife_oplist(metaid);
        int ret;

        if (!ops)
                return -ENOENT;
        ret = __add_metainfo(ops, ife, metaid, metaval, len, false, exists);
        if (ret)
                /*put back what find_ife_oplist took */
                module_put(ops->owner);
        return ret;
}

static int use_all_metadata(struct tcf_ife_info *ife, bool exists)
{
        struct tcf_meta_ops *o;
        int rc = 0;
        int installed = 0;

        read_lock(&ife_mod_lock);
        list_for_each_entry(o, &ifeoplist, list) {
                rc = add_metainfo_and_get_ops(o, ife, o->metaid, exists);
                if (rc == 0)
                        installed += 1;
        }
        read_unlock(&ife_mod_lock);

        if (installed)
                return 0;
        else
                return -EINVAL;
}

static int dump_metalist(struct sk_buff *skb, struct tcf_ife_info *ife)
{
        struct tcf_meta_info *e;
        struct nlattr *nest;
        unsigned char *b = skb_tail_pointer(skb);
        int total_encoded = 0;

        /*can only happen on decode */
        if (list_empty(&ife->metalist))
                return 0;

        nest = nla_nest_start_noflag(skb, TCA_IFE_METALST);
        if (!nest)
                goto out_nlmsg_trim;

        list_for_each_entry(e, &ife->metalist, metalist) {
                if (!e->ops->get(skb, e))
                        total_encoded += 1;
        }

        if (!total_encoded)
                goto out_nlmsg_trim;

        nla_nest_end(skb, nest);

        return 0;

out_nlmsg_trim:
        nlmsg_trim(skb, b);
        return -1;
}

/* under ife->tcf_lock */
static void _tcf_ife_cleanup(struct tc_action *a)
{
        struct tcf_ife_info *ife = to_ife(a);
        struct tcf_meta_info *e, *n;

        list_for_each_entry_safe(e, n, &ife->metalist, metalist) {
                list_del(&e->metalist);
                if (e->metaval) {
                        if (e->ops->release)
                                e->ops->release(e);
                        else
                                kfree(e->metaval);
                }
                module_put(e->ops->owner);
                kfree(e);
        }
}

static void tcf_ife_cleanup(struct tc_action *a)
{
        struct tcf_ife_info *ife = to_ife(a);
        struct tcf_ife_params *p;

        spin_lock_bh(&ife->tcf_lock);
        _tcf_ife_cleanup(a);
        spin_unlock_bh(&ife->tcf_lock);

        p = rcu_dereference_protected(ife->params, 1);
        if (p)
                kfree_rcu(p, rcu);
}

static int load_metalist(struct nlattr **tb, bool rtnl_held)
{
        int i;

        for (i = 1; i < max_metacnt; i++) {
                if (tb[i]) {
                        void *val = nla_data(tb[i]);
                        int len = nla_len(tb[i]);
                        int rc;

                        rc = load_metaops_and_vet(i, val, len, rtnl_held);
                        if (rc != 0)
                                return rc;
                }
        }

        return 0;
}

static int populate_metalist(struct tcf_ife_info *ife, struct nlattr **tb,
                             bool exists, bool rtnl_held)
{
        int len = 0;
        int rc = 0;
        int i = 0;
        void *val;

        for (i = 1; i < max_metacnt; i++) {
                if (tb[i]) {
                        val = nla_data(tb[i]);
                        len = nla_len(tb[i]);

                        rc = add_metainfo(ife, i, val, len, exists);
                        if (rc)
                                return rc;
                }
        }

        return rc;
}

static int tcf_ife_init(struct net *net, struct nlattr *nla,
                        struct nlattr *est, struct tc_action **a,
                        struct tcf_proto *tp, u32 flags,
                        struct netlink_ext_ack *extack)
{
        struct tc_action_net *tn = net_generic(net, act_ife_ops.net_id);
        bool bind = flags & TCA_ACT_FLAGS_BIND;
        struct nlattr *tb[TCA_IFE_MAX + 1];
        struct nlattr *tb2[IFE_META_MAX + 1];
        struct tcf_chain *goto_ch = NULL;
        struct tcf_ife_params *p;
        struct tcf_ife_info *ife;
        u16 ife_type = ETH_P_IFE;
        struct tc_ife *parm;
        u8 *daddr = NULL;
        u8 *saddr = NULL;
        bool exists = false;
        int ret = 0;
        u32 index;
        int err;

        if (!nla) {
                NL_SET_ERR_MSG_MOD(extack, "IFE requires attributes to be passed");
                return -EINVAL;
        }

        err = nla_parse_nested_deprecated(tb, TCA_IFE_MAX, nla, ife_policy,
                                          NULL);
        if (err < 0)
                return err;

        if (!tb[TCA_IFE_PARMS])
                return -EINVAL;

        parm = nla_data(tb[TCA_IFE_PARMS]);

        /* IFE_DECODE is 0 and indicates the opposite of IFE_ENCODE because
         * they cannot run as the same time. Check on all other values which
         * are not supported right now.
         */
        if (parm->flags & ~IFE_ENCODE)
                return -EINVAL;

        p = kzalloc(sizeof(*p), GFP_KERNEL);
        if (!p)
                return -ENOMEM;

        if (tb[TCA_IFE_METALST]) {
                err = nla_parse_nested_deprecated(tb2, IFE_META_MAX,
                                                  tb[TCA_IFE_METALST], NULL,
                                                  NULL);
                if (err) {
                        kfree(p);
                        return err;
                }
                err = load_metalist(tb2, !(flags & TCA_ACT_FLAGS_NO_RTNL));
                if (err) {
                        kfree(p);
                        return err;
                }
        }

        index = parm->index;
        err = tcf_idr_check_alloc(tn, &index, a, bind);
        if (err < 0) {
                kfree(p);
                return err;
        }
        exists = err;
        if (exists && bind) {
                kfree(p);
                return ACT_P_BOUND;
        }

        if (!exists) {
                ret = tcf_idr_create(tn, index, est, a, &act_ife_ops,
                                     bind, true, flags);
                if (ret) {
                        tcf_idr_cleanup(tn, index);
                        kfree(p);
                        return ret;
                }
                ret = ACT_P_CREATED;
        } else if (!(flags & TCA_ACT_FLAGS_REPLACE)) {
                tcf_idr_release(*a, bind);
                kfree(p);
                return -EEXIST;
        }

        ife = to_ife(*a);
        if (ret == ACT_P_CREATED)
                INIT_LIST_HEAD(&ife->metalist);

        err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
        if (err < 0)
                goto release_idr;

        p->flags = parm->flags;

        if (parm->flags & IFE_ENCODE) {
                if (tb[TCA_IFE_TYPE])
                        ife_type = nla_get_u16(tb[TCA_IFE_TYPE]);
                if (tb[TCA_IFE_DMAC])
                        daddr = nla_data(tb[TCA_IFE_DMAC]);
                if (tb[TCA_IFE_SMAC])
                        saddr = nla_data(tb[TCA_IFE_SMAC]);
        }

        if (parm->flags & IFE_ENCODE) {
                if (daddr)
                        ether_addr_copy(p->eth_dst, daddr);
                else
                        eth_zero_addr(p->eth_dst);

                if (saddr)
                        ether_addr_copy(p->eth_src, saddr);
                else
                        eth_zero_addr(p->eth_src);

                p->eth_type = ife_type;
        }

        if (tb[TCA_IFE_METALST]) {
                err = populate_metalist(ife, tb2, exists,
                                        !(flags & TCA_ACT_FLAGS_NO_RTNL));
                if (err)
                        goto metadata_parse_err;
        } else {
                /* if no passed metadata allow list or passed allow-all
                 * then here we process by adding as many supported metadatum
                 * as we can. You better have at least one else we are
                 * going to bail out
                 */
                err = use_all_metadata(ife, exists);
                if (err)
                        goto metadata_parse_err;
        }

        if (exists)
                spin_lock_bh(&ife->tcf_lock);
        /* protected by tcf_lock when modifying existing action */
        goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
        p = rcu_replace_pointer(ife->params, p, 1);

        if (exists)
                spin_unlock_bh(&ife->tcf_lock);
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
        if (p)
                kfree_rcu(p, rcu);

        return ret;
metadata_parse_err:
        if (goto_ch)
                tcf_chain_put_by_act(goto_ch);
release_idr:
        kfree(p);
        tcf_idr_release(*a, bind);
        return err;
}

static int tcf_ife_dump(struct sk_buff *skb, struct tc_action *a, int bind,
                        int ref)
{
        unsigned char *b = skb_tail_pointer(skb);
        struct tcf_ife_info *ife = to_ife(a);
        struct tcf_ife_params *p;
        struct tc_ife opt = {
                .index = ife->tcf_index,
                .refcnt = refcount_read(&ife->tcf_refcnt) - ref,
                .bindcnt = atomic_read(&ife->tcf_bindcnt) - bind,
        };
        struct tcf_t t;

        spin_lock_bh(&ife->tcf_lock);
        opt.action = ife->tcf_action;
        p = rcu_dereference_protected(ife->params,
                                      lockdep_is_held(&ife->tcf_lock));
        opt.flags = p->flags;

        if (nla_put(skb, TCA_IFE_PARMS, sizeof(opt), &opt))
                goto nla_put_failure;

        tcf_tm_dump(&t, &ife->tcf_tm);
        if (nla_put_64bit(skb, TCA_IFE_TM, sizeof(t), &t, TCA_IFE_PAD))
                goto nla_put_failure;

        if (!is_zero_ether_addr(p->eth_dst)) {
                if (nla_put(skb, TCA_IFE_DMAC, ETH_ALEN, p->eth_dst))
                        goto nla_put_failure;
        }

        if (!is_zero_ether_addr(p->eth_src)) {
                if (nla_put(skb, TCA_IFE_SMAC, ETH_ALEN, p->eth_src))
                        goto nla_put_failure;
        }

        if (nla_put(skb, TCA_IFE_TYPE, 2, &p->eth_type))
                goto nla_put_failure;

        if (dump_metalist(skb, ife)) {
                /*ignore failure to dump metalist */
                pr_info("Failed to dump metalist\n");
        }

        spin_unlock_bh(&ife->tcf_lock);
        return skb->len;

nla_put_failure:
        spin_unlock_bh(&ife->tcf_lock);
        nlmsg_trim(skb, b);
        return -1;
}

static int find_decode_metaid(struct sk_buff *skb, struct tcf_ife_info *ife,
                              u16 metaid, u16 mlen, void *mdata)
{
        struct tcf_meta_info *e;

        /* XXX: use hash to speed up */
        list_for_each_entry(e, &ife->metalist, metalist) {
                if (metaid == e->metaid) {
                        if (e->ops) {
                                /* We check for decode presence already */
                                return e->ops->decode(skb, mdata, mlen);
                        }
                }
        }

        return -ENOENT;
}

static int tcf_ife_decode(struct sk_buff *skb, const struct tc_action *a,
                          struct tcf_result *res)
{
        struct tcf_ife_info *ife = to_ife(a);
        int action = ife->tcf_action;
        u8 *ifehdr_end;
        u8 *tlv_data;
        u16 metalen;

        bstats_update(this_cpu_ptr(ife->common.cpu_bstats), skb);
        tcf_lastuse_update(&ife->tcf_tm);

        if (skb_at_tc_ingress(skb))
                skb_push(skb, skb->dev->hard_header_len);

        tlv_data = ife_decode(skb, &metalen);
        if (unlikely(!tlv_data)) {
                qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats));
                return TC_ACT_SHOT;
        }

        ifehdr_end = tlv_data + metalen;
        for (; tlv_data < ifehdr_end; tlv_data = ife_tlv_meta_next(tlv_data)) {
                u8 *curr_data;
                u16 mtype;
                u16 dlen;

                curr_data = ife_tlv_meta_decode(tlv_data, ifehdr_end, &mtype,
                                                &dlen, NULL);
                if (!curr_data) {
                        qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats));
                        return TC_ACT_SHOT;
                }

                if (find_decode_metaid(skb, ife, mtype, dlen, curr_data)) {
                        /* abuse overlimits to count when we receive metadata
                         * but dont have an ops for it
                         */
                        pr_info_ratelimited("Unknown metaid %d dlen %d\n",
                                            mtype, dlen);
                        qstats_overlimit_inc(this_cpu_ptr(ife->common.cpu_qstats));
                }
        }

        if (WARN_ON(tlv_data != ifehdr_end)) {
                qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats));
                return TC_ACT_SHOT;
        }

        skb->protocol = eth_type_trans(skb, skb->dev);
        skb_reset_network_header(skb);

        return action;
}

/*XXX: check if we can do this at install time instead of current
 * send data path
**/
static int ife_get_sz(struct sk_buff *skb, struct tcf_ife_info *ife)
{
        struct tcf_meta_info *e, *n;
        int tot_run_sz = 0, run_sz = 0;

        list_for_each_entry_safe(e, n, &ife->metalist, metalist) {
                if (e->ops->check_presence) {
                        run_sz = e->ops->check_presence(skb, e);
                        tot_run_sz += run_sz;
                }
        }

        return tot_run_sz;
}

static int tcf_ife_encode(struct sk_buff *skb, const struct tc_action *a,
                          struct tcf_result *res, struct tcf_ife_params *p)
{
        struct tcf_ife_info *ife = to_ife(a);
        int action = ife->tcf_action;
        struct ethhdr *oethh;        /* outer ether header */
        struct tcf_meta_info *e;
        /*
           OUTERHDR:TOTMETALEN:{TLVHDR:Metadatum:TLVHDR..}:ORIGDATA
           where ORIGDATA = original ethernet header ...
         */
        u16 metalen = ife_get_sz(skb, ife);
        int hdrm = metalen + skb->dev->hard_header_len + IFE_METAHDRLEN;
        unsigned int skboff = 0;
        int new_len = skb->len + hdrm;
        bool exceed_mtu = false;
        void *ife_meta;
        int err = 0;

        if (!skb_at_tc_ingress(skb)) {
                if (new_len > skb->dev->mtu)
                        exceed_mtu = true;
        }

        bstats_update(this_cpu_ptr(ife->common.cpu_bstats), skb);
        tcf_lastuse_update(&ife->tcf_tm);

        if (!metalen) {                /* no metadata to send */
                /* abuse overlimits to count when we allow packet
                 * with no metadata
                 */
                qstats_overlimit_inc(this_cpu_ptr(ife->common.cpu_qstats));
                return action;
        }
        /* could be stupid policy setup or mtu config
         * so lets be conservative.. */
        if ((action == TC_ACT_SHOT) || exceed_mtu) {
                qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats));
                return TC_ACT_SHOT;
        }

        if (skb_at_tc_ingress(skb))
                skb_push(skb, skb->dev->hard_header_len);

        ife_meta = ife_encode(skb, metalen);

        spin_lock(&ife->tcf_lock);

        /* XXX: we dont have a clever way of telling encode to
         * not repeat some of the computations that are done by
         * ops->presence_check...
         */
        list_for_each_entry(e, &ife->metalist, metalist) {
                if (e->ops->encode) {
                        err = e->ops->encode(skb, (void *)(ife_meta + skboff),
                                             e);
                }
                if (err < 0) {
                        /* too corrupt to keep around if overwritten */
                        spin_unlock(&ife->tcf_lock);
                        qstats_drop_inc(this_cpu_ptr(ife->common.cpu_qstats));
                        return TC_ACT_SHOT;
                }
                skboff += err;
        }
        spin_unlock(&ife->tcf_lock);
        oethh = (struct ethhdr *)skb->data;

        if (!is_zero_ether_addr(p->eth_src))
                ether_addr_copy(oethh->h_source, p->eth_src);
        if (!is_zero_ether_addr(p->eth_dst))
                ether_addr_copy(oethh->h_dest, p->eth_dst);
        oethh->h_proto = htons(p->eth_type);

        if (skb_at_tc_ingress(skb))
                skb_pull(skb, skb->dev->hard_header_len);

        return action;
}

TC_INDIRECT_SCOPE int tcf_ife_act(struct sk_buff *skb,
                                  const struct tc_action *a,
                                  struct tcf_result *res)
{
        struct tcf_ife_info *ife = to_ife(a);
        struct tcf_ife_params *p;
        int ret;

        p = rcu_dereference_bh(ife->params);
        if (p->flags & IFE_ENCODE) {
                ret = tcf_ife_encode(skb, a, res, p);
                return ret;
        }

        return tcf_ife_decode(skb, a, res);
}

static struct tc_action_ops act_ife_ops = {
        .kind = "ife",
        .id = TCA_ID_IFE,
        .owner = THIS_MODULE,
        .act = tcf_ife_act,
        .dump = tcf_ife_dump,
        .cleanup = tcf_ife_cleanup,
        .init = tcf_ife_init,
        .size =        sizeof(struct tcf_ife_info),
};
MODULE_ALIAS_NET_ACT("ife");

static __net_init int ife_init_net(struct net *net)
{
        struct tc_action_net *tn = net_generic(net, act_ife_ops.net_id);

        return tc_action_net_init(net, tn, &act_ife_ops);
}

static void __net_exit ife_exit_net(struct list_head *net_list)
{
        tc_action_net_exit(net_list, act_ife_ops.net_id);
}

static struct pernet_operations ife_net_ops = {
        .init = ife_init_net,
        .exit_batch = ife_exit_net,
        .id   = &act_ife_ops.net_id,
        .size = sizeof(struct tc_action_net),
};

static int __init ife_init_module(void)
{
        return tcf_register_action(&act_ife_ops, &ife_net_ops);
}

static void __exit ife_cleanup_module(void)
{
        tcf_unregister_action(&act_ife_ops, &ife_net_ops);
}

module_init(ife_init_module);
module_exit(ife_cleanup_module);

MODULE_AUTHOR("Jamal Hadi Salim(2015)");
MODULE_DESCRIPTION("Inter-FE LFB action");
MODULE_LICENSE("GPL");

























































































































































































































































































































































































































































































































































































































































  197 

  183 

































































































































































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_FB_H
#define _LINUX_FB_H

#include <uapi/linux/fb.h>

#define FBIO_CURSOR            _IOWR('F', 0x08, struct fb_cursor_user)

#include <linux/mutex.h>
#include <linux/printk.h>
#include <linux/refcount.h>
#include <linux/types.h>
#include <linux/workqueue.h>

#include <asm/video.h>

struct backlight_device;
struct device;
struct device_node;
struct fb_info;
struct file;
struct i2c_adapter;
struct inode;
struct module;
struct notifier_block;
struct page;
struct videomode;
struct vm_area_struct;

/* Definitions below are used in the parsed monitor specs */
#define FB_DPMS_ACTIVE_OFF        1
#define FB_DPMS_SUSPEND                2
#define FB_DPMS_STANDBY                4

#define FB_DISP_DDI                1
#define FB_DISP_ANA_700_300        2
#define FB_DISP_ANA_714_286        4
#define FB_DISP_ANA_1000_400        8
#define FB_DISP_ANA_700_000        16

#define FB_DISP_MONO                32
#define FB_DISP_RGB                64
#define FB_DISP_MULTI                128
#define FB_DISP_UNKNOWN                256

#define FB_SIGNAL_NONE                0
#define FB_SIGNAL_BLANK_BLANK        1
#define FB_SIGNAL_SEPARATE        2
#define FB_SIGNAL_COMPOSITE        4
#define FB_SIGNAL_SYNC_ON_GREEN        8
#define FB_SIGNAL_SERRATION_ON        16

#define FB_MISC_PRIM_COLOR        1
#define FB_MISC_1ST_DETAIL        2        /* First Detailed Timing is preferred */
#define FB_MISC_HDMI                4
struct fb_chroma {
        __u32 redx;        /* in fraction of 1024 */
        __u32 greenx;
        __u32 bluex;
        __u32 whitex;
        __u32 redy;
        __u32 greeny;
        __u32 bluey;
        __u32 whitey;
};

struct fb_monspecs {
        struct fb_chroma chroma;
        struct fb_videomode *modedb;        /* mode database */
        __u8  manufacturer[4];                /* Manufacturer */
        __u8  monitor[14];                /* Monitor String */
        __u8  serial_no[14];                /* Serial Number */
        __u8  ascii[14];                /* ? */
        __u32 modedb_len;                /* mode database length */
        __u32 model;                        /* Monitor Model */
        __u32 serial;                        /* Serial Number - Integer */
        __u32 year;                        /* Year manufactured */
        __u32 week;                        /* Week Manufactured */
        __u32 hfmin;                        /* hfreq lower limit (Hz) */
        __u32 hfmax;                        /* hfreq upper limit (Hz) */
        __u32 dclkmin;                        /* pixelclock lower limit (Hz) */
        __u32 dclkmax;                        /* pixelclock upper limit (Hz) */
        __u16 input;                        /* display type - see FB_DISP_* */
        __u16 dpms;                        /* DPMS support - see FB_DPMS_ */
        __u16 signal;                        /* Signal Type - see FB_SIGNAL_* */
        __u16 vfmin;                        /* vfreq lower limit (Hz) */
        __u16 vfmax;                        /* vfreq upper limit (Hz) */
        __u16 gamma;                        /* Gamma - in fractions of 100 */
        __u16 gtf        : 1;                /* supports GTF */
        __u16 misc;                        /* Misc flags - see FB_MISC_* */
        __u8  version;                        /* EDID version... */
        __u8  revision;                        /* ...and revision */
        __u8  max_x;                        /* Maximum horizontal size (cm) */
        __u8  max_y;                        /* Maximum vertical size (cm) */
};

struct fb_cmap_user {
        __u32 start;                        /* First entry        */
        __u32 len;                        /* Number of entries */
        __u16 __user *red;                /* Red values        */
        __u16 __user *green;
        __u16 __user *blue;
        __u16 __user *transp;                /* transparency, can be NULL */
};

struct fb_image_user {
        __u32 dx;                        /* Where to place image */
        __u32 dy;
        __u32 width;                        /* Size of image */
        __u32 height;
        __u32 fg_color;                        /* Only used when a mono bitmap */
        __u32 bg_color;
        __u8  depth;                        /* Depth of the image */
        const char __user *data;        /* Pointer to image data */
        struct fb_cmap_user cmap;        /* color map info */
};

struct fb_cursor_user {
        __u16 set;                        /* what to set */
        __u16 enable;                        /* cursor on/off */
        __u16 rop;                        /* bitop operation */
        const char __user *mask;        /* cursor mask bits */
        struct fbcurpos hot;                /* cursor hot spot */
        struct fb_image_user image;        /* Cursor image */
};

/*
 * Register/unregister for framebuffer events
 */

/*        The resolution of the passed in fb_info about to change */
#define FB_EVENT_MODE_CHANGE                0x01

#ifdef CONFIG_GUMSTIX_AM200EPD
/* only used by mach-pxa/am200epd.c */
#define FB_EVENT_FB_REGISTERED          0x05
#define FB_EVENT_FB_UNREGISTERED        0x06
#endif

/*      A display blank is requested       */
#define FB_EVENT_BLANK                  0x09

struct fb_event {
        struct fb_info *info;
        void *data;
};

/*        Enough for the VT console needs, see its max_font_width/height */
#define FB_MAX_BLIT_WIDTH        64
#define FB_MAX_BLIT_HEIGHT        128

struct fb_blit_caps {
        DECLARE_BITMAP(x, FB_MAX_BLIT_WIDTH);
        DECLARE_BITMAP(y, FB_MAX_BLIT_HEIGHT);
        u32 len;
        u32 flags;
};

#ifdef CONFIG_FB_NOTIFY
extern int fb_register_client(struct notifier_block *nb);
extern int fb_unregister_client(struct notifier_block *nb);
extern int fb_notifier_call_chain(unsigned long val, void *v);
#else
static inline int fb_register_client(struct notifier_block *nb)
{
        return 0;
};

static inline int fb_unregister_client(struct notifier_block *nb)
{
        return 0;
};

static inline int fb_notifier_call_chain(unsigned long val, void *v)
{
        return 0;
};
#endif

/*
 * Pixmap structure definition
 *
 * The purpose of this structure is to translate data
 * from the hardware independent format of fbdev to what
 * format the hardware needs.
 */

#define FB_PIXMAP_DEFAULT 1     /* used internally by fbcon */
#define FB_PIXMAP_SYSTEM  2     /* memory is in system RAM  */
#define FB_PIXMAP_IO      4     /* memory is iomapped       */
#define FB_PIXMAP_SYNC    256   /* set if GPU can DMA       */

struct fb_pixmap {
        u8  *addr;                /* pointer to memory                        */
        u32 size;                /* size of buffer in bytes                */
        u32 offset;                /* current offset to buffer                */
        u32 buf_align;                /* byte alignment of each bitmap        */
        u32 scan_align;                /* alignment per scanline                */
        u32 access_align;        /* alignment per read/write (bits)        */
        u32 flags;                /* see FB_PIXMAP_*                        */
                                /* supported bit block dimensions        */
                                /* Format: test_bit(width - 1, blit_x)        */
                                /*           test_bit(height - 1, blit_y)        */
                                /* if zero, will be set to full (all)        */
        DECLARE_BITMAP(blit_x, FB_MAX_BLIT_WIDTH);
        DECLARE_BITMAP(blit_y, FB_MAX_BLIT_HEIGHT);
        /* access methods */
        void (*writeio)(struct fb_info *info, void __iomem *dst, void *src, unsigned int size);
        void (*readio) (struct fb_info *info, void *dst, void __iomem *src, unsigned int size);
};

#ifdef CONFIG_FB_DEFERRED_IO
struct fb_deferred_io_pageref {
        struct page *page;
        unsigned long offset;
        /* private */
        struct list_head list;
};

struct fb_deferred_io {
        /* delay between mkwrite and deferred handler */
        unsigned long delay;
        bool sort_pagereflist; /* sort pagelist by offset */
        int open_count; /* number of opened files; protected by fb_info lock */
        struct mutex lock; /* mutex that protects the pageref list */
        struct list_head pagereflist; /* list of pagerefs for touched pages */
        /* callback */
        struct page *(*get_page)(struct fb_info *info, unsigned long offset);
        void (*deferred_io)(struct fb_info *info, struct list_head *pagelist);
};
#endif

/*
 * Frame buffer operations
 *
 * LOCKING NOTE: those functions must _ALL_ be called with the console
 * semaphore held, this is the only suitable locking mechanism we have
 * in 2.6. Some may be called at interrupt time at this point though.
 *
 * The exception to this is the debug related hooks.  Putting the fb
 * into a debug state (e.g. flipping to the kernel console) and restoring
 * it must be done in a lock-free manner, so low level drivers should
 * keep track of the initial console (if applicable) and may need to
 * perform direct, unlocked hardware writes in these hooks.
 */

struct fb_ops {
        /* open/release and usage marking */
        struct module *owner;
        int (*fb_open)(struct fb_info *info, int user);
        int (*fb_release)(struct fb_info *info, int user);

        /* For framebuffers with strange non linear layouts or that do not
         * work with normal memory mapped access
         */
        ssize_t (*fb_read)(struct fb_info *info, char __user *buf,
                           size_t count, loff_t *ppos);
        ssize_t (*fb_write)(struct fb_info *info, const char __user *buf,
                            size_t count, loff_t *ppos);

        /* checks var and eventually tweaks it to something supported,
         * DO NOT MODIFY PAR */
        int (*fb_check_var)(struct fb_var_screeninfo *var, struct fb_info *info);

        /* set the video mode according to info->var */
        int (*fb_set_par)(struct fb_info *info);

        /* set color register */
        int (*fb_setcolreg)(unsigned regno, unsigned red, unsigned green,
                            unsigned blue, unsigned transp, struct fb_info *info);

        /* set color registers in batch */
        int (*fb_setcmap)(struct fb_cmap *cmap, struct fb_info *info);

        /* blank display */
        int (*fb_blank)(int blank, struct fb_info *info);

        /* pan display */
        int (*fb_pan_display)(struct fb_var_screeninfo *var, struct fb_info *info);

        /* Draws a rectangle */
        void (*fb_fillrect) (struct fb_info *info, const struct fb_fillrect *rect);
        /* Copy data from area to another */
        void (*fb_copyarea) (struct fb_info *info, const struct fb_copyarea *region);
        /* Draws a image to the display */
        void (*fb_imageblit) (struct fb_info *info, const struct fb_image *image);

        /* Draws cursor */
        int (*fb_cursor) (struct fb_info *info, struct fb_cursor *cursor);

        /* wait for blit idle, optional */
        int (*fb_sync)(struct fb_info *info);

        /* perform fb specific ioctl (optional) */
        int (*fb_ioctl)(struct fb_info *info, unsigned int cmd,
                        unsigned long arg);

        /* Handle 32bit compat ioctl (optional) */
        int (*fb_compat_ioctl)(struct fb_info *info, unsigned cmd,
                        unsigned long arg);

        /* perform fb specific mmap */
        int (*fb_mmap)(struct fb_info *info, struct vm_area_struct *vma);

        /* get capability given var */
        void (*fb_get_caps)(struct fb_info *info, struct fb_blit_caps *caps,
                            struct fb_var_screeninfo *var);

        /* teardown any resources to do with this framebuffer */
        void (*fb_destroy)(struct fb_info *info);

        /* called at KDB enter and leave time to prepare the console */
        int (*fb_debug_enter)(struct fb_info *info);
        int (*fb_debug_leave)(struct fb_info *info);
};

#ifdef CONFIG_FB_TILEBLITTING
#define FB_TILE_CURSOR_NONE        0
#define FB_TILE_CURSOR_UNDERLINE   1
#define FB_TILE_CURSOR_LOWER_THIRD 2
#define FB_TILE_CURSOR_LOWER_HALF  3
#define FB_TILE_CURSOR_TWO_THIRDS  4
#define FB_TILE_CURSOR_BLOCK       5

struct fb_tilemap {
        __u32 width;                /* width of each tile in pixels */
        __u32 height;               /* height of each tile in scanlines */
        __u32 depth;                /* color depth of each tile */
        __u32 length;               /* number of tiles in the map */
        const __u8 *data;           /* actual tile map: a bitmap array, packed
                                       to the nearest byte */
};

struct fb_tilerect {
        __u32 sx;                   /* origin in the x-axis */
        __u32 sy;                   /* origin in the y-axis */
        __u32 width;                /* number of tiles in the x-axis */
        __u32 height;               /* number of tiles in the y-axis */
        __u32 index;                /* what tile to use: index to tile map */
        __u32 fg;                   /* foreground color */
        __u32 bg;                   /* background color */
        __u32 rop;                  /* raster operation */
};

struct fb_tilearea {
        __u32 sx;                   /* source origin in the x-axis */
        __u32 sy;                   /* source origin in the y-axis */
        __u32 dx;                   /* destination origin in the x-axis */
        __u32 dy;                   /* destination origin in the y-axis */
        __u32 width;                /* number of tiles in the x-axis */
        __u32 height;               /* number of tiles in the y-axis */
};

struct fb_tileblit {
        __u32 sx;                   /* origin in the x-axis */
        __u32 sy;                   /* origin in the y-axis */
        __u32 width;                /* number of tiles in the x-axis */
        __u32 height;               /* number of tiles in the y-axis */
        __u32 fg;                   /* foreground color */
        __u32 bg;                   /* background color */
        __u32 length;               /* number of tiles to draw */
        __u32 *indices;             /* array of indices to tile map */
};

struct fb_tilecursor {
        __u32 sx;                   /* cursor position in the x-axis */
        __u32 sy;                   /* cursor position in the y-axis */
        __u32 mode;                 /* 0 = erase, 1 = draw */
        __u32 shape;                /* see FB_TILE_CURSOR_* */
        __u32 fg;                   /* foreground color */
        __u32 bg;                   /* background color */
};

struct fb_tile_ops {
        /* set tile characteristics */
        void (*fb_settile)(struct fb_info *info, struct fb_tilemap *map);

        /* all dimensions from hereon are in terms of tiles */

        /* move a rectangular region of tiles from one area to another*/
        void (*fb_tilecopy)(struct fb_info *info, struct fb_tilearea *area);
        /* fill a rectangular region with a tile */
        void (*fb_tilefill)(struct fb_info *info, struct fb_tilerect *rect);
        /* copy an array of tiles */
        void (*fb_tileblit)(struct fb_info *info, struct fb_tileblit *blit);
        /* cursor */
        void (*fb_tilecursor)(struct fb_info *info,
                              struct fb_tilecursor *cursor);
        /* get maximum length of the tile map */
        int (*fb_get_tilemax)(struct fb_info *info);
};
#endif /* CONFIG_FB_TILEBLITTING */

/* FBINFO_* = fb_info.flags bit flags */
#define FBINFO_HWACCEL_DISABLED        0x0002
        /* When FBINFO_HWACCEL_DISABLED is set:
         *  Hardware acceleration is turned off.  Software implementations
         *  of required functions (copyarea(), fillrect(), and imageblit())
         *  takes over; acceleration engine should be in a quiescent state */

/* hints */
#define FBINFO_VIRTFB                0x0004 /* FB is System RAM, not device. */
#define FBINFO_PARTIAL_PAN_OK        0x0040 /* otw use pan only for double-buffering */
#define FBINFO_READS_FAST        0x0080 /* soft-copy faster than rendering */

/* hardware supported ops */
/*  semantics: when a bit is set, it indicates that the operation is
 *   accelerated by hardware.
 *  required functions will still work even if the bit is not set.
 *  optional functions may not even exist if the flag bit is not set.
 */
#define FBINFO_HWACCEL_NONE                0x0000
#define FBINFO_HWACCEL_COPYAREA                0x0100 /* required */
#define FBINFO_HWACCEL_FILLRECT                0x0200 /* required */
#define FBINFO_HWACCEL_IMAGEBLIT        0x0400 /* required */
#define FBINFO_HWACCEL_ROTATE                0x0800 /* optional */
#define FBINFO_HWACCEL_XPAN                0x1000 /* optional */
#define FBINFO_HWACCEL_YPAN                0x2000 /* optional */
#define FBINFO_HWACCEL_YWRAP                0x4000 /* optional */

#define FBINFO_MISC_TILEBLITTING       0x20000 /* use tile blitting */

/* A driver may set this flag to indicate that it does want a set_par to be
 * called every time when fbcon_switch is executed. The advantage is that with
 * this flag set you can really be sure that set_par is always called before
 * any of the functions dependent on the correct hardware state or altering
 * that state, even if you are using some broken X releases. The disadvantage
 * is that it introduces unwanted delays to every console switch if set_par
 * is slow. It is a good idea to try this flag in the drivers initialization
 * code whenever there is a bug report related to switching between X and the
 * framebuffer console.
 */
#define FBINFO_MISC_ALWAYS_SETPAR   0x40000

/*
 * Host and GPU endianness differ.
 */
#define FBINFO_FOREIGN_ENDIAN        0x100000
/*
 * Big endian math. This is the same flags as above, but with different
 * meaning, it is set by the fb subsystem depending FOREIGN_ENDIAN flag
 * and host endianness. Drivers should not use this flag.
 */
#define FBINFO_BE_MATH  0x100000
/*
 * Hide smem_start in the FBIOGET_FSCREENINFO IOCTL. This is used by modern DRM
 * drivers to stop userspace from trying to share buffers behind the kernel's
 * back. Instead dma-buf based buffer sharing should be used.
 */
#define FBINFO_HIDE_SMEM_START  0x200000


struct fb_info {
        refcount_t count;
        int node;
        int flags;
        /*
         * -1 by default, set to a FB_ROTATE_* value by the driver, if it knows
         * a lcd is not mounted upright and fbcon should rotate to compensate.
         */
        int fbcon_rotate_hint;
        struct mutex lock;                /* Lock for open/release/ioctl funcs */
        struct mutex mm_lock;                /* Lock for fb_mmap and smem_* fields */
        struct fb_var_screeninfo var;        /* Current var */
        struct fb_fix_screeninfo fix;        /* Current fix */
        struct fb_monspecs monspecs;        /* Current Monitor specs */
        struct fb_pixmap pixmap;        /* Image hardware mapper */
        struct fb_pixmap sprite;        /* Cursor hardware mapper */
        struct fb_cmap cmap;                /* Current cmap */
        struct list_head modelist;      /* mode list */
        struct fb_videomode *mode;        /* current mode */

#if IS_ENABLED(CONFIG_FB_BACKLIGHT)
        /* assigned backlight device */
        /* set before framebuffer registration,
           remove after unregister */
        struct backlight_device *bl_dev;

        /* Backlight level curve */
        struct mutex bl_curve_mutex;
        u8 bl_curve[FB_BACKLIGHT_LEVELS];
#endif
#ifdef CONFIG_FB_DEFERRED_IO
        struct delayed_work deferred_work;
        unsigned long npagerefs;
        struct fb_deferred_io_pageref *pagerefs;
        struct fb_deferred_io *fbdefio;
#endif

        const struct fb_ops *fbops;
        struct device *device;                /* This is the parent */
#if defined(CONFIG_FB_DEVICE)
        struct device *dev;                /* This is this fb device */
#endif
        int class_flag;                    /* private sysfs flags */
#ifdef CONFIG_FB_TILEBLITTING
        struct fb_tile_ops *tileops;    /* Tile Blitting */
#endif
        union {
                char __iomem *screen_base;        /* Virtual address */
                char *screen_buffer;
        };
        unsigned long screen_size;        /* Amount of ioremapped VRAM or 0 */
        void *pseudo_palette;                /* Fake palette of 16 colors */
#define FBINFO_STATE_RUNNING        0
#define FBINFO_STATE_SUSPENDED        1
        u32 state;                        /* Hardware state i.e suspend */
        void *fbcon_par;                /* fbcon use-only private area */
        /* From here on everything is device dependent */
        void *par;

        bool skip_vt_switch; /* no VT switch on suspend/resume required */
        bool skip_panic; /* Do not write to the fb after a panic */
};

/* This will go away
 * fbset currently hacks in FB_ACCELF_TEXT into var.accel_flags
 * when it wants to turn the acceleration engine on.  This is
 * really a separate operation, and should be modified via sysfs.
 *  But for now, we leave it broken with the following define
 */
#define STUPID_ACCELF_TEXT_SHIT

#define FB_LEFT_POS(p, bpp)          (fb_be_math(p) ? (32 - (bpp)) : 0)
#define FB_SHIFT_HIGH(p, val, bits)  (fb_be_math(p) ? (val) >> (bits) : \
                                                      (val) << (bits))
#define FB_SHIFT_LOW(p, val, bits)   (fb_be_math(p) ? (val) << (bits) : \
                                                      (val) >> (bits))

    /*
     *  `Generic' versions of the frame buffer device operations
     */

extern int fb_set_var(struct fb_info *info, struct fb_var_screeninfo *var);
extern int fb_pan_display(struct fb_info *info, struct fb_var_screeninfo *var);
extern int fb_blank(struct fb_info *info, int blank);

/*
 * Helpers for framebuffers in I/O memory
 */

extern void cfb_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
extern void cfb_copyarea(struct fb_info *info, const struct fb_copyarea *area);
extern void cfb_imageblit(struct fb_info *info, const struct fb_image *image);
extern ssize_t fb_io_read(struct fb_info *info, char __user *buf,
                          size_t count, loff_t *ppos);
extern ssize_t fb_io_write(struct fb_info *info, const char __user *buf,
                           size_t count, loff_t *ppos);
int fb_io_mmap(struct fb_info *info, struct vm_area_struct *vma);

#define __FB_DEFAULT_IOMEM_OPS_RDWR \
        .fb_read        = fb_io_read, \
        .fb_write        = fb_io_write

#define __FB_DEFAULT_IOMEM_OPS_DRAW \
        .fb_fillrect        = cfb_fillrect, \
        .fb_copyarea        = cfb_copyarea, \
        .fb_imageblit        = cfb_imageblit

#define __FB_DEFAULT_IOMEM_OPS_MMAP \
        .fb_mmap        = fb_io_mmap

#define FB_DEFAULT_IOMEM_OPS \
        __FB_DEFAULT_IOMEM_OPS_RDWR, \
        __FB_DEFAULT_IOMEM_OPS_DRAW, \
        __FB_DEFAULT_IOMEM_OPS_MMAP

/*
 * Helpers for framebuffers in system memory
 */

extern void sys_fillrect(struct fb_info *info, const struct fb_fillrect *rect);
extern void sys_copyarea(struct fb_info *info, const struct fb_copyarea *area);
extern void sys_imageblit(struct fb_info *info, const struct fb_image *image);
extern ssize_t fb_sys_read(struct fb_info *info, char __user *buf,
                           size_t count, loff_t *ppos);
extern ssize_t fb_sys_write(struct fb_info *info, const char __user *buf,
                            size_t count, loff_t *ppos);

#define __FB_DEFAULT_SYSMEM_OPS_RDWR \
        .fb_read        = fb_sys_read, \
        .fb_write        = fb_sys_write

#define __FB_DEFAULT_SYSMEM_OPS_DRAW \
        .fb_fillrect        = sys_fillrect, \
        .fb_copyarea        = sys_copyarea, \
        .fb_imageblit        = sys_imageblit

/*
 * Helpers for framebuffers in DMA-able memory
 */

#define __FB_DEFAULT_DMAMEM_OPS_RDWR \
        .fb_read        = fb_sys_read, \
        .fb_write        = fb_sys_write

#define __FB_DEFAULT_DMAMEM_OPS_DRAW \
        .fb_fillrect        = sys_fillrect, \
        .fb_copyarea        = sys_copyarea, \
        .fb_imageblit        = sys_imageblit

/* fbmem.c */
extern int register_framebuffer(struct fb_info *fb_info);
extern void unregister_framebuffer(struct fb_info *fb_info);
extern int devm_register_framebuffer(struct device *dev, struct fb_info *fb_info);
extern char* fb_get_buffer_offset(struct fb_info *info, struct fb_pixmap *buf, u32 size);
extern void fb_pad_unaligned_buffer(u8 *dst, u32 d_pitch, u8 *src, u32 idx,
                                u32 height, u32 shift_high, u32 shift_low, u32 mod);
extern void fb_pad_aligned_buffer(u8 *dst, u32 d_pitch, u8 *src, u32 s_pitch, u32 height);
extern void fb_set_suspend(struct fb_info *info, int state);
extern int fb_get_color_depth(struct fb_var_screeninfo *var,
                              struct fb_fix_screeninfo *fix);
extern int fb_get_options(const char *name, char **option);
extern int fb_new_modelist(struct fb_info *info);

static inline void lock_fb_info(struct fb_info *info)
{
        mutex_lock(&info->lock);
}

static inline void unlock_fb_info(struct fb_info *info)
{
        mutex_unlock(&info->lock);
}

static inline void __fb_pad_aligned_buffer(u8 *dst, u32 d_pitch,
                                           u8 *src, u32 s_pitch, u32 height)
{
        u32 i, j;

        d_pitch -= s_pitch;

        for (i = height; i--; ) {
                /* s_pitch is a few bytes at the most, memcpy is suboptimal */
                for (j = 0; j < s_pitch; j++)
                        *dst++ = *src++;
                dst += d_pitch;
        }
}

/* fb_defio.c */
int fb_deferred_io_mmap(struct fb_info *info, struct vm_area_struct *vma);
extern int  fb_deferred_io_init(struct fb_info *info);
extern void fb_deferred_io_open(struct fb_info *info,
                                struct inode *inode,
                                struct file *file);
extern void fb_deferred_io_release(struct fb_info *info);
extern void fb_deferred_io_cleanup(struct fb_info *info);
extern int fb_deferred_io_fsync(struct file *file, loff_t start,
                                loff_t end, int datasync);

/*
 * Generate callbacks for deferred I/O
 */

#define __FB_GEN_DEFAULT_DEFERRED_OPS_RDWR(__prefix, __damage_range, __mode) \
        static ssize_t __prefix ## _defio_read(struct fb_info *info, char __user *buf, \
                                               size_t count, loff_t *ppos) \
        { \
                return fb_ ## __mode ## _read(info, buf, count, ppos); \
        } \
        static ssize_t __prefix ## _defio_write(struct fb_info *info, const char __user *buf, \
                                                size_t count, loff_t *ppos) \
        { \
                unsigned long offset = *ppos; \
                ssize_t ret = fb_ ## __mode ## _write(info, buf, count, ppos); \
                if (ret > 0) \
                        __damage_range(info, offset, ret); \
                return ret; \
        }

#define __FB_GEN_DEFAULT_DEFERRED_OPS_DRAW(__prefix, __damage_area, __mode) \
        static void __prefix ## _defio_fillrect(struct fb_info *info, \
                                                const struct fb_fillrect *rect) \
        { \
                __mode ## _fillrect(info, rect); \
                __damage_area(info, rect->dx, rect->dy, rect->width, rect->height); \
        } \
        static void __prefix ## _defio_copyarea(struct fb_info *info, \
                                                const struct fb_copyarea *area) \
        { \
                __mode ## _copyarea(info, area); \
                __damage_area(info, area->dx, area->dy, area->width, area->height); \
        } \
        static void __prefix ## _defio_imageblit(struct fb_info *info, \
                                                 const struct fb_image *image) \
        { \
                __mode ## _imageblit(info, image); \
                __damage_area(info, image->dx, image->dy, image->width, image->height); \
        }

#define FB_GEN_DEFAULT_DEFERRED_IOMEM_OPS(__prefix, __damage_range, __damage_area) \
        __FB_GEN_DEFAULT_DEFERRED_OPS_RDWR(__prefix, __damage_range, io) \
        __FB_GEN_DEFAULT_DEFERRED_OPS_DRAW(__prefix, __damage_area, cfb)

#define FB_GEN_DEFAULT_DEFERRED_SYSMEM_OPS(__prefix, __damage_range, __damage_area) \
        __FB_GEN_DEFAULT_DEFERRED_OPS_RDWR(__prefix, __damage_range, sys) \
        __FB_GEN_DEFAULT_DEFERRED_OPS_DRAW(__prefix, __damage_area, sys)

#define FB_GEN_DEFAULT_DEFERRED_DMAMEM_OPS(__prefix, __damage_range, __damage_area) \
        __FB_GEN_DEFAULT_DEFERRED_OPS_RDWR(__prefix, __damage_range, sys) \
        __FB_GEN_DEFAULT_DEFERRED_OPS_DRAW(__prefix, __damage_area, sys)

/*
 * Initializes struct fb_ops for deferred I/O.
 */

#define __FB_DEFAULT_DEFERRED_OPS_RDWR(__prefix) \
        .fb_read        = __prefix ## _defio_read, \
        .fb_write        = __prefix ## _defio_write

#define __FB_DEFAULT_DEFERRED_OPS_DRAW(__prefix) \
        .fb_fillrect        = __prefix ## _defio_fillrect, \
        .fb_copyarea        = __prefix ## _defio_copyarea, \
        .fb_imageblit        = __prefix ## _defio_imageblit

#define __FB_DEFAULT_DEFERRED_OPS_MMAP(__prefix) \
        .fb_mmap        = fb_deferred_io_mmap

#define FB_DEFAULT_DEFERRED_OPS(__prefix) \
        __FB_DEFAULT_DEFERRED_OPS_RDWR(__prefix), \
        __FB_DEFAULT_DEFERRED_OPS_DRAW(__prefix), \
        __FB_DEFAULT_DEFERRED_OPS_MMAP(__prefix)

static inline bool fb_be_math(struct fb_info *info)
{
#ifdef CONFIG_FB_FOREIGN_ENDIAN
#if defined(CONFIG_FB_BOTH_ENDIAN)
        return info->flags & FBINFO_BE_MATH;
#elif defined(CONFIG_FB_BIG_ENDIAN)
        return true;
#elif defined(CONFIG_FB_LITTLE_ENDIAN)
        return false;
#endif /* CONFIG_FB_BOTH_ENDIAN */
#else
#ifdef __BIG_ENDIAN
        return true;
#else
        return false;
#endif /* __BIG_ENDIAN */
#endif /* CONFIG_FB_FOREIGN_ENDIAN */
}

extern struct fb_info *framebuffer_alloc(size_t size, struct device *dev);
extern void framebuffer_release(struct fb_info *info);
extern void fb_bl_default_curve(struct fb_info *fb_info, u8 off, u8 min, u8 max);

#if IS_ENABLED(CONFIG_FB_BACKLIGHT)
struct backlight_device *fb_bl_device(struct fb_info *info);
#else
static inline struct backlight_device *fb_bl_device(struct fb_info *info)
{
        return NULL;
}
#endif

/* fbmon.c */
#define FB_MAXTIMINGS                0
#define FB_VSYNCTIMINGS                1
#define FB_HSYNCTIMINGS                2
#define FB_DCLKTIMINGS                3
#define FB_IGNOREMON                0x100

#define FB_MODE_IS_UNKNOWN        0
#define FB_MODE_IS_DETAILED        1
#define FB_MODE_IS_STANDARD        2
#define FB_MODE_IS_VESA                4
#define FB_MODE_IS_CALCULATED        8
#define FB_MODE_IS_FIRST        16
#define FB_MODE_IS_FROM_VAR     32

extern int fbmon_dpms(const struct fb_info *fb_info);
extern int fb_get_mode(int flags, u32 val, struct fb_var_screeninfo *var,
                       struct fb_info *info);
extern int fb_validate_mode(const struct fb_var_screeninfo *var,
                            struct fb_info *info);
extern int fb_parse_edid(unsigned char *edid, struct fb_var_screeninfo *var);
extern const unsigned char *fb_firmware_edid(struct device *device);
extern void fb_edid_to_monspecs(unsigned char *edid,
                                struct fb_monspecs *specs);
extern void fb_destroy_modedb(struct fb_videomode *modedb);
extern int fb_find_mode_cvt(struct fb_videomode *mode, int margins, int rb);
extern unsigned char *fb_ddc_read(struct i2c_adapter *adapter);

extern int of_get_fb_videomode(struct device_node *np,
                               struct fb_videomode *fb,
                               int index);
extern int fb_videomode_from_videomode(const struct videomode *vm,
                                       struct fb_videomode *fbmode);

/* modedb.c */
#define VESA_MODEDB_SIZE 43
#define DMT_SIZE 0x50

extern void fb_var_to_videomode(struct fb_videomode *mode,
                                const struct fb_var_screeninfo *var);
extern void fb_videomode_to_var(struct fb_var_screeninfo *var,
                                const struct fb_videomode *mode);
extern int fb_mode_is_equal(const struct fb_videomode *mode1,
                            const struct fb_videomode *mode2);
extern int fb_add_videomode(const struct fb_videomode *mode,
                            struct list_head *head);
extern void fb_delete_videomode(const struct fb_videomode *mode,
                                struct list_head *head);
extern const struct fb_videomode *fb_match_mode(const struct fb_var_screeninfo *var,
                                                struct list_head *head);
extern const struct fb_videomode *fb_find_best_mode(const struct fb_var_screeninfo *var,
                                                    struct list_head *head);
extern const struct fb_videomode *fb_find_nearest_mode(const struct fb_videomode *mode,
                                                       struct list_head *head);
extern void fb_destroy_modelist(struct list_head *head);
extern void fb_videomode_to_modelist(const struct fb_videomode *modedb, int num,
                                     struct list_head *head);
extern const struct fb_videomode *fb_find_best_display(const struct fb_monspecs *specs,
                                                       struct list_head *head);

/* fbcmap.c */
extern int fb_alloc_cmap(struct fb_cmap *cmap, int len, int transp);
extern int fb_alloc_cmap_gfp(struct fb_cmap *cmap, int len, int transp, gfp_t flags);
extern void fb_dealloc_cmap(struct fb_cmap *cmap);
extern int fb_copy_cmap(const struct fb_cmap *from, struct fb_cmap *to);
extern int fb_cmap_to_user(const struct fb_cmap *from, struct fb_cmap_user *to);
extern int fb_set_cmap(struct fb_cmap *cmap, struct fb_info *fb_info);
extern int fb_set_user_cmap(struct fb_cmap_user *cmap, struct fb_info *fb_info);
extern const struct fb_cmap *fb_default_cmap(int len);
extern void fb_invert_cmaps(void);

struct fb_videomode {
        const char *name;        /* optional */
        u32 refresh;                /* optional */
        u32 xres;
        u32 yres;
        u32 pixclock;
        u32 left_margin;
        u32 right_margin;
        u32 upper_margin;
        u32 lower_margin;
        u32 hsync_len;
        u32 vsync_len;
        u32 sync;
        u32 vmode;
        u32 flag;
};

struct dmt_videomode {
        u32 dmt_id;
        u32 std_2byte_code;
        u32 cvt_3byte_code;
        const struct fb_videomode *mode;
};

extern const struct fb_videomode vesa_modes[];
extern const struct dmt_videomode dmt_modes[];

struct fb_modelist {
        struct list_head list;
        struct fb_videomode mode;
};

extern int fb_find_mode(struct fb_var_screeninfo *var,
                        struct fb_info *info, const char *mode_option,
                        const struct fb_videomode *db,
                        unsigned int dbsize,
                        const struct fb_videomode *default_mode,
                        unsigned int default_bpp);

bool fb_modesetting_disabled(const char *drvname);

/*
 * Convenience logging macros
 */

#define fb_err(fb_info, fmt, ...)                                        \
        pr_err("fb%d: " fmt, (fb_info)->node, ##__VA_ARGS__)
#define fb_notice(info, fmt, ...)                                        \
        pr_notice("fb%d: " fmt, (fb_info)->node, ##__VA_ARGS__)
#define fb_warn(fb_info, fmt, ...)                                        \
        pr_warn("fb%d: " fmt, (fb_info)->node, ##__VA_ARGS__)
#define fb_info(fb_info, fmt, ...)                                        \
        pr_info("fb%d: " fmt, (fb_info)->node, ##__VA_ARGS__)
#define fb_dbg(fb_info, fmt, ...)                                        \
        pr_debug("fb%d: " fmt, (fb_info)->node, ##__VA_ARGS__)

#define fb_warn_once(fb_info, fmt, ...)                                        \
        pr_warn_once("fb%d: " fmt, (fb_info)->node, ##__VA_ARGS__)

#define fb_WARN_ONCE(fb_info, condition, fmt, ...) \
        WARN_ONCE(condition, "fb%d: " fmt, (fb_info)->node, ##__VA_ARGS__)
#define fb_WARN_ON_ONCE(fb_info, x) \
        fb_WARN_ONCE(fb_info, (x), "%s", "fb_WARN_ON_ONCE(" __stringify(x) ")")

#endif /* _LINUX_FB_H */






























































































  765 


   17 
   17 



   13 














































































































  749 


   16 
   16 


   13 

























































































































































































































































  517 










   77 



    6 









  168 











  771 



  769 

  756 


















  828 










   55 





  751 




   19 
    2 






  773 





  773 

  743 
   18 









  761 











  762 










  761 



















  762 









  762 
  760 


  758 





  761 







  760 














































































   11 



















   11 



   11 

    9 





   11 


   11 





   11 









































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (C) 2005,2006,2007,2008 IBM Corporation
 *
 * Authors:
 * Mimi Zohar <zohar@us.ibm.com>
 * Kylene Hall <kjhall@us.ibm.com>
 *
 * File: ima_crypto.c
 *        Calculates md5/sha1 file hash, template hash, boot-aggreate hash
 */

#include <linux/kernel.h>
#include <linux/moduleparam.h>
#include <linux/ratelimit.h>
#include <linux/file.h>
#include <linux/crypto.h>
#include <linux/scatterlist.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <crypto/hash.h>

#include "ima.h"

/* minimum file size for ahash use */
static unsigned long ima_ahash_minsize;
module_param_named(ahash_minsize, ima_ahash_minsize, ulong, 0644);
MODULE_PARM_DESC(ahash_minsize, "Minimum file size for ahash use");

/* default is 0 - 1 page. */
static int ima_maxorder;
static unsigned int ima_bufsize = PAGE_SIZE;

static int param_set_bufsize(const char *val, const struct kernel_param *kp)
{
        unsigned long long size;
        int order;

        size = memparse(val, NULL);
        order = get_order(size);
        if (order > MAX_PAGE_ORDER)
                return -EINVAL;
        ima_maxorder = order;
        ima_bufsize = PAGE_SIZE << order;
        return 0;
}

static const struct kernel_param_ops param_ops_bufsize = {
        .set = param_set_bufsize,
        .get = param_get_uint,
};
#define param_check_bufsize(name, p) __param_check(name, p, unsigned int)

module_param_named(ahash_bufsize, ima_bufsize, bufsize, 0644);
MODULE_PARM_DESC(ahash_bufsize, "Maximum ahash buffer size");

static struct crypto_shash *ima_shash_tfm;
static struct crypto_ahash *ima_ahash_tfm;

int ima_sha1_idx __ro_after_init;
int ima_hash_algo_idx __ro_after_init;
/*
 * Additional number of slots reserved, as needed, for SHA1
 * and IMA default algo.
 */
int ima_extra_slots __ro_after_init;

struct ima_algo_desc *ima_algo_array __ro_after_init;

static int __init ima_init_ima_crypto(void)
{
        long rc;

        ima_shash_tfm = crypto_alloc_shash(hash_algo_name[ima_hash_algo], 0, 0);
        if (IS_ERR(ima_shash_tfm)) {
                rc = PTR_ERR(ima_shash_tfm);
                pr_err("Can not allocate %s (reason: %ld)\n",
                       hash_algo_name[ima_hash_algo], rc);
                return rc;
        }
        pr_info("Allocated hash algorithm: %s\n",
                hash_algo_name[ima_hash_algo]);
        return 0;
}

static struct crypto_shash *ima_alloc_tfm(enum hash_algo algo)
{
        struct crypto_shash *tfm = ima_shash_tfm;
        int rc, i;

        if (algo < 0 || algo >= HASH_ALGO__LAST)
                algo = ima_hash_algo;

        if (algo == ima_hash_algo)
                return tfm;

        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++)
                if (ima_algo_array[i].tfm && ima_algo_array[i].algo == algo)
                        return ima_algo_array[i].tfm;

        tfm = crypto_alloc_shash(hash_algo_name[algo], 0, 0);
        if (IS_ERR(tfm)) {
                rc = PTR_ERR(tfm);
                pr_err("Can not allocate %s (reason: %d)\n",
                       hash_algo_name[algo], rc);
        }
        return tfm;
}

int __init ima_init_crypto(void)
{
        enum hash_algo algo;
        long rc;
        int i;

        rc = ima_init_ima_crypto();
        if (rc)
                return rc;

        ima_sha1_idx = -1;
        ima_hash_algo_idx = -1;

        for (i = 0; i < NR_BANKS(ima_tpm_chip); i++) {
                algo = ima_tpm_chip->allocated_banks[i].crypto_id;
                if (algo == HASH_ALGO_SHA1)
                        ima_sha1_idx = i;

                if (algo == ima_hash_algo)
                        ima_hash_algo_idx = i;
        }

        if (ima_sha1_idx < 0) {
                ima_sha1_idx = NR_BANKS(ima_tpm_chip) + ima_extra_slots++;
                if (ima_hash_algo == HASH_ALGO_SHA1)
                        ima_hash_algo_idx = ima_sha1_idx;
        }

        if (ima_hash_algo_idx < 0)
                ima_hash_algo_idx = NR_BANKS(ima_tpm_chip) + ima_extra_slots++;

        ima_algo_array = kcalloc(NR_BANKS(ima_tpm_chip) + ima_extra_slots,
                                 sizeof(*ima_algo_array), GFP_KERNEL);
        if (!ima_algo_array) {
                rc = -ENOMEM;
                goto out;
        }

        for (i = 0; i < NR_BANKS(ima_tpm_chip); i++) {
                algo = ima_tpm_chip->allocated_banks[i].crypto_id;
                ima_algo_array[i].algo = algo;

                /* unknown TPM algorithm */
                if (algo == HASH_ALGO__LAST)
                        continue;

                if (algo == ima_hash_algo) {
                        ima_algo_array[i].tfm = ima_shash_tfm;
                        continue;
                }

                ima_algo_array[i].tfm = ima_alloc_tfm(algo);
                if (IS_ERR(ima_algo_array[i].tfm)) {
                        if (algo == HASH_ALGO_SHA1) {
                                rc = PTR_ERR(ima_algo_array[i].tfm);
                                ima_algo_array[i].tfm = NULL;
                                goto out_array;
                        }

                        ima_algo_array[i].tfm = NULL;
                }
        }

        if (ima_sha1_idx >= NR_BANKS(ima_tpm_chip)) {
                if (ima_hash_algo == HASH_ALGO_SHA1) {
                        ima_algo_array[ima_sha1_idx].tfm = ima_shash_tfm;
                } else {
                        ima_algo_array[ima_sha1_idx].tfm =
                                                ima_alloc_tfm(HASH_ALGO_SHA1);
                        if (IS_ERR(ima_algo_array[ima_sha1_idx].tfm)) {
                                rc = PTR_ERR(ima_algo_array[ima_sha1_idx].tfm);
                                goto out_array;
                        }
                }

                ima_algo_array[ima_sha1_idx].algo = HASH_ALGO_SHA1;
        }

        if (ima_hash_algo_idx >= NR_BANKS(ima_tpm_chip) &&
            ima_hash_algo_idx != ima_sha1_idx) {
                ima_algo_array[ima_hash_algo_idx].tfm = ima_shash_tfm;
                ima_algo_array[ima_hash_algo_idx].algo = ima_hash_algo;
        }

        return 0;
out_array:
        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++) {
                if (!ima_algo_array[i].tfm ||
                    ima_algo_array[i].tfm == ima_shash_tfm)
                        continue;

                crypto_free_shash(ima_algo_array[i].tfm);
        }
        kfree(ima_algo_array);
out:
        crypto_free_shash(ima_shash_tfm);
        return rc;
}

static void ima_free_tfm(struct crypto_shash *tfm)
{
        int i;

        if (tfm == ima_shash_tfm)
                return;

        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++)
                if (ima_algo_array[i].tfm == tfm)
                        return;

        crypto_free_shash(tfm);
}

/**
 * ima_alloc_pages() - Allocate contiguous pages.
 * @max_size:       Maximum amount of memory to allocate.
 * @allocated_size: Returned size of actual allocation.
 * @last_warn:      Should the min_size allocation warn or not.
 *
 * Tries to do opportunistic allocation for memory first trying to allocate
 * max_size amount of memory and then splitting that until zero order is
 * reached. Allocation is tried without generating allocation warnings unless
 * last_warn is set. Last_warn set affects only last allocation of zero order.
 *
 * By default, ima_maxorder is 0 and it is equivalent to kmalloc(GFP_KERNEL)
 *
 * Return pointer to allocated memory, or NULL on failure.
 */
static void *ima_alloc_pages(loff_t max_size, size_t *allocated_size,
                             int last_warn)
{
        void *ptr;
        int order = ima_maxorder;
        gfp_t gfp_mask = __GFP_RECLAIM | __GFP_NOWARN | __GFP_NORETRY;

        if (order)
                order = min(get_order(max_size), order);

        for (; order; order--) {
                ptr = (void *)__get_free_pages(gfp_mask, order);
                if (ptr) {
                        *allocated_size = PAGE_SIZE << order;
                        return ptr;
                }
        }

        /* order is zero - one page */

        gfp_mask = GFP_KERNEL;

        if (!last_warn)
                gfp_mask |= __GFP_NOWARN;

        ptr = (void *)__get_free_pages(gfp_mask, 0);
        if (ptr) {
                *allocated_size = PAGE_SIZE;
                return ptr;
        }

        *allocated_size = 0;
        return NULL;
}

/**
 * ima_free_pages() - Free pages allocated by ima_alloc_pages().
 * @ptr:  Pointer to allocated pages.
 * @size: Size of allocated buffer.
 */
static void ima_free_pages(void *ptr, size_t size)
{
        if (!ptr)
                return;
        free_pages((unsigned long)ptr, get_order(size));
}

static struct crypto_ahash *ima_alloc_atfm(enum hash_algo algo)
{
        struct crypto_ahash *tfm = ima_ahash_tfm;
        int rc;

        if (algo < 0 || algo >= HASH_ALGO__LAST)
                algo = ima_hash_algo;

        if (algo != ima_hash_algo || !tfm) {
                tfm = crypto_alloc_ahash(hash_algo_name[algo], 0, 0);
                if (!IS_ERR(tfm)) {
                        if (algo == ima_hash_algo)
                                ima_ahash_tfm = tfm;
                } else {
                        rc = PTR_ERR(tfm);
                        pr_err("Can not allocate %s (reason: %d)\n",
                               hash_algo_name[algo], rc);
                }
        }
        return tfm;
}

static void ima_free_atfm(struct crypto_ahash *tfm)
{
        if (tfm != ima_ahash_tfm)
                crypto_free_ahash(tfm);
}

static inline int ahash_wait(int err, struct crypto_wait *wait)
{

        err = crypto_wait_req(err, wait);

        if (err)
                pr_crit_ratelimited("ahash calculation failed: err: %d\n", err);

        return err;
}

static int ima_calc_file_hash_atfm(struct file *file,
                                   struct ima_digest_data *hash,
                                   struct crypto_ahash *tfm)
{
        loff_t i_size, offset;
        char *rbuf[2] = { NULL, };
        int rc, rbuf_len, active = 0, ahash_rc = 0;
        struct ahash_request *req;
        struct scatterlist sg[1];
        struct crypto_wait wait;
        size_t rbuf_size[2];

        hash->length = crypto_ahash_digestsize(tfm);

        req = ahash_request_alloc(tfm, GFP_KERNEL);
        if (!req)
                return -ENOMEM;

        crypto_init_wait(&wait);
        ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
                                   CRYPTO_TFM_REQ_MAY_SLEEP,
                                   crypto_req_done, &wait);

        rc = ahash_wait(crypto_ahash_init(req), &wait);
        if (rc)
                goto out1;

        i_size = i_size_read(file_inode(file));

        if (i_size == 0)
                goto out2;

        /*
         * Try to allocate maximum size of memory.
         * Fail if even a single page cannot be allocated.
         */
        rbuf[0] = ima_alloc_pages(i_size, &rbuf_size[0], 1);
        if (!rbuf[0]) {
                rc = -ENOMEM;
                goto out1;
        }

        /* Only allocate one buffer if that is enough. */
        if (i_size > rbuf_size[0]) {
                /*
                 * Try to allocate secondary buffer. If that fails fallback to
                 * using single buffering. Use previous memory allocation size
                 * as baseline for possible allocation size.
                 */
                rbuf[1] = ima_alloc_pages(i_size - rbuf_size[0],
                                          &rbuf_size[1], 0);
        }

        for (offset = 0; offset < i_size; offset += rbuf_len) {
                if (!rbuf[1] && offset) {
                        /* Not using two buffers, and it is not the first
                         * read/request, wait for the completion of the
                         * previous ahash_update() request.
                         */
                        rc = ahash_wait(ahash_rc, &wait);
                        if (rc)
                                goto out3;
                }
                /* read buffer */
                rbuf_len = min_t(loff_t, i_size - offset, rbuf_size[active]);
                rc = integrity_kernel_read(file, offset, rbuf[active],
                                           rbuf_len);
                if (rc != rbuf_len) {
                        if (rc >= 0)
                                rc = -EINVAL;
                        /*
                         * Forward current rc, do not overwrite with return value
                         * from ahash_wait()
                         */
                        ahash_wait(ahash_rc, &wait);
                        goto out3;
                }

                if (rbuf[1] && offset) {
                        /* Using two buffers, and it is not the first
                         * read/request, wait for the completion of the
                         * previous ahash_update() request.
                         */
                        rc = ahash_wait(ahash_rc, &wait);
                        if (rc)
                                goto out3;
                }

                sg_init_one(&sg[0], rbuf[active], rbuf_len);
                ahash_request_set_crypt(req, sg, NULL, rbuf_len);

                ahash_rc = crypto_ahash_update(req);

                if (rbuf[1])
                        active = !active; /* swap buffers, if we use two */
        }
        /* wait for the last update request to complete */
        rc = ahash_wait(ahash_rc, &wait);
out3:
        ima_free_pages(rbuf[0], rbuf_size[0]);
        ima_free_pages(rbuf[1], rbuf_size[1]);
out2:
        if (!rc) {
                ahash_request_set_crypt(req, NULL, hash->digest, 0);
                rc = ahash_wait(crypto_ahash_final(req), &wait);
        }
out1:
        ahash_request_free(req);
        return rc;
}

static int ima_calc_file_ahash(struct file *file, struct ima_digest_data *hash)
{
        struct crypto_ahash *tfm;
        int rc;

        tfm = ima_alloc_atfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = ima_calc_file_hash_atfm(file, hash, tfm);

        ima_free_atfm(tfm);

        return rc;
}

static int ima_calc_file_hash_tfm(struct file *file,
                                  struct ima_digest_data *hash,
                                  struct crypto_shash *tfm)
{
        loff_t i_size, offset = 0;
        char *rbuf;
        int rc;
        SHASH_DESC_ON_STACK(shash, tfm);

        shash->tfm = tfm;

        hash->length = crypto_shash_digestsize(tfm);

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        i_size = i_size_read(file_inode(file));

        if (i_size == 0)
                goto out;

        rbuf = kzalloc(PAGE_SIZE, GFP_KERNEL);
        if (!rbuf)
                return -ENOMEM;

        while (offset < i_size) {
                int rbuf_len;

                rbuf_len = integrity_kernel_read(file, offset, rbuf, PAGE_SIZE);
                if (rbuf_len < 0) {
                        rc = rbuf_len;
                        break;
                }
                if (rbuf_len == 0) {        /* unexpected EOF */
                        rc = -EINVAL;
                        break;
                }
                offset += rbuf_len;

                rc = crypto_shash_update(shash, rbuf, rbuf_len);
                if (rc)
                        break;
        }
        kfree(rbuf);
out:
        if (!rc)
                rc = crypto_shash_final(shash, hash->digest);
        return rc;
}

static int ima_calc_file_shash(struct file *file, struct ima_digest_data *hash)
{
        struct crypto_shash *tfm;
        int rc;

        tfm = ima_alloc_tfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = ima_calc_file_hash_tfm(file, hash, tfm);

        ima_free_tfm(tfm);

        return rc;
}

/*
 * ima_calc_file_hash - calculate file hash
 *
 * Asynchronous hash (ahash) allows using HW acceleration for calculating
 * a hash. ahash performance varies for different data sizes on different
 * crypto accelerators. shash performance might be better for smaller files.
 * The 'ima.ahash_minsize' module parameter allows specifying the best
 * minimum file size for using ahash on the system.
 *
 * If the ima.ahash_minsize parameter is not specified, this function uses
 * shash for the hash calculation.  If ahash fails, it falls back to using
 * shash.
 */
int ima_calc_file_hash(struct file *file, struct ima_digest_data *hash)
{
        loff_t i_size;
        int rc;
        struct file *f = file;
        bool new_file_instance = false;

        /*
         * For consistency, fail file's opened with the O_DIRECT flag on
         * filesystems mounted with/without DAX option.
         */
        if (file->f_flags & O_DIRECT) {
                hash->length = hash_digest_size[ima_hash_algo];
                hash->algo = ima_hash_algo;
                return -EINVAL;
        }

        /* Open a new file instance in O_RDONLY if we cannot read */
        if (!(file->f_mode & FMODE_READ)) {
                int flags = file->f_flags & ~(O_WRONLY | O_APPEND |
                                O_TRUNC | O_CREAT | O_NOCTTY | O_EXCL);
                flags |= O_RDONLY;
                f = dentry_open(&file->f_path, flags, file->f_cred);
                if (IS_ERR(f))
                        return PTR_ERR(f);

                new_file_instance = true;
        }

        i_size = i_size_read(file_inode(f));

        if (ima_ahash_minsize && i_size >= ima_ahash_minsize) {
                rc = ima_calc_file_ahash(f, hash);
                if (!rc)
                        goto out;
        }

        rc = ima_calc_file_shash(f, hash);
out:
        if (new_file_instance)
                fput(f);
        return rc;
}

/*
 * Calculate the hash of template data
 */
static int ima_calc_field_array_hash_tfm(struct ima_field_data *field_data,
                                         struct ima_template_entry *entry,
                                         int tfm_idx)
{
        SHASH_DESC_ON_STACK(shash, ima_algo_array[tfm_idx].tfm);
        struct ima_template_desc *td = entry->template_desc;
        int num_fields = entry->template_desc->num_fields;
        int rc, i;

        shash->tfm = ima_algo_array[tfm_idx].tfm;

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        for (i = 0; i < num_fields; i++) {
                u8 buffer[IMA_EVENT_NAME_LEN_MAX + 1] = { 0 };
                u8 *data_to_hash = field_data[i].data;
                u32 datalen = field_data[i].len;
                u32 datalen_to_hash = !ima_canonical_fmt ?
                                datalen : (__force u32)cpu_to_le32(datalen);

                if (strcmp(td->name, IMA_TEMPLATE_IMA_NAME) != 0) {
                        rc = crypto_shash_update(shash,
                                                (const u8 *) &datalen_to_hash,
                                                sizeof(datalen_to_hash));
                        if (rc)
                                break;
                } else if (strcmp(td->fields[i]->field_id, "n") == 0) {
                        memcpy(buffer, data_to_hash, datalen);
                        data_to_hash = buffer;
                        datalen = IMA_EVENT_NAME_LEN_MAX + 1;
                }
                rc = crypto_shash_update(shash, data_to_hash, datalen);
                if (rc)
                        break;
        }

        if (!rc)
                rc = crypto_shash_final(shash, entry->digests[tfm_idx].digest);

        return rc;
}

int ima_calc_field_array_hash(struct ima_field_data *field_data,
                              struct ima_template_entry *entry)
{
        u16 alg_id;
        int rc, i;

        rc = ima_calc_field_array_hash_tfm(field_data, entry, ima_sha1_idx);
        if (rc)
                return rc;

        entry->digests[ima_sha1_idx].alg_id = TPM_ALG_SHA1;

        for (i = 0; i < NR_BANKS(ima_tpm_chip) + ima_extra_slots; i++) {
                if (i == ima_sha1_idx)
                        continue;

                if (i < NR_BANKS(ima_tpm_chip)) {
                        alg_id = ima_tpm_chip->allocated_banks[i].alg_id;
                        entry->digests[i].alg_id = alg_id;
                }

                /* for unmapped TPM algorithms digest is still a padded SHA1 */
                if (!ima_algo_array[i].tfm) {
                        memcpy(entry->digests[i].digest,
                               entry->digests[ima_sha1_idx].digest,
                               TPM_DIGEST_SIZE);
                        continue;
                }

                rc = ima_calc_field_array_hash_tfm(field_data, entry, i);
                if (rc)
                        return rc;
        }
        return rc;
}

static int calc_buffer_ahash_atfm(const void *buf, loff_t len,
                                  struct ima_digest_data *hash,
                                  struct crypto_ahash *tfm)
{
        struct ahash_request *req;
        struct scatterlist sg;
        struct crypto_wait wait;
        int rc, ahash_rc = 0;

        hash->length = crypto_ahash_digestsize(tfm);

        req = ahash_request_alloc(tfm, GFP_KERNEL);
        if (!req)
                return -ENOMEM;

        crypto_init_wait(&wait);
        ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG |
                                   CRYPTO_TFM_REQ_MAY_SLEEP,
                                   crypto_req_done, &wait);

        rc = ahash_wait(crypto_ahash_init(req), &wait);
        if (rc)
                goto out;

        sg_init_one(&sg, buf, len);
        ahash_request_set_crypt(req, &sg, NULL, len);

        ahash_rc = crypto_ahash_update(req);

        /* wait for the update request to complete */
        rc = ahash_wait(ahash_rc, &wait);
        if (!rc) {
                ahash_request_set_crypt(req, NULL, hash->digest, 0);
                rc = ahash_wait(crypto_ahash_final(req), &wait);
        }
out:
        ahash_request_free(req);
        return rc;
}

static int calc_buffer_ahash(const void *buf, loff_t len,
                             struct ima_digest_data *hash)
{
        struct crypto_ahash *tfm;
        int rc;

        tfm = ima_alloc_atfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = calc_buffer_ahash_atfm(buf, len, hash, tfm);

        ima_free_atfm(tfm);

        return rc;
}

static int calc_buffer_shash_tfm(const void *buf, loff_t size,
                                struct ima_digest_data *hash,
                                struct crypto_shash *tfm)
{
        SHASH_DESC_ON_STACK(shash, tfm);
        unsigned int len;
        int rc;

        shash->tfm = tfm;

        hash->length = crypto_shash_digestsize(tfm);

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        while (size) {
                len = size < PAGE_SIZE ? size : PAGE_SIZE;
                rc = crypto_shash_update(shash, buf, len);
                if (rc)
                        break;
                buf += len;
                size -= len;
        }

        if (!rc)
                rc = crypto_shash_final(shash, hash->digest);
        return rc;
}

static int calc_buffer_shash(const void *buf, loff_t len,
                             struct ima_digest_data *hash)
{
        struct crypto_shash *tfm;
        int rc;

        tfm = ima_alloc_tfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        rc = calc_buffer_shash_tfm(buf, len, hash, tfm);

        ima_free_tfm(tfm);
        return rc;
}

int ima_calc_buffer_hash(const void *buf, loff_t len,
                         struct ima_digest_data *hash)
{
        int rc;

        if (ima_ahash_minsize && len >= ima_ahash_minsize) {
                rc = calc_buffer_ahash(buf, len, hash);
                if (!rc)
                        return 0;
        }

        return calc_buffer_shash(buf, len, hash);
}

static void ima_pcrread(u32 idx, struct tpm_digest *d)
{
        if (!ima_tpm_chip)
                return;

        if (tpm_pcr_read(ima_tpm_chip, idx, d) != 0)
                pr_err("Error Communicating to TPM chip\n");
}

/*
 * The boot_aggregate is a cumulative hash over TPM registers 0 - 7.  With
 * TPM 1.2 the boot_aggregate was based on reading the SHA1 PCRs, but with
 * TPM 2.0 hash agility, TPM chips could support multiple TPM PCR banks,
 * allowing firmware to configure and enable different banks.
 *
 * Knowing which TPM bank is read to calculate the boot_aggregate digest
 * needs to be conveyed to a verifier.  For this reason, use the same
 * hash algorithm for reading the TPM PCRs as for calculating the boot
 * aggregate digest as stored in the measurement list.
 */
static int ima_calc_boot_aggregate_tfm(char *digest, u16 alg_id,
                                       struct crypto_shash *tfm)
{
        struct tpm_digest d = { .alg_id = alg_id, .digest = {0} };
        int rc;
        u32 i;
        SHASH_DESC_ON_STACK(shash, tfm);

        shash->tfm = tfm;

        pr_devel("calculating the boot-aggregate based on TPM bank: %04x\n",
                 d.alg_id);

        rc = crypto_shash_init(shash);
        if (rc != 0)
                return rc;

        /* cumulative digest over TPM registers 0-7 */
        for (i = TPM_PCR0; i < TPM_PCR8; i++) {
                ima_pcrread(i, &d);
                /* now accumulate with current aggregate */
                rc = crypto_shash_update(shash, d.digest,
                                         crypto_shash_digestsize(tfm));
                if (rc != 0)
                        return rc;
        }
        /*
         * Extend cumulative digest over TPM registers 8-9, which contain
         * measurement for the kernel command line (reg. 8) and image (reg. 9)
         * in a typical PCR allocation. Registers 8-9 are only included in
         * non-SHA1 boot_aggregate digests to avoid ambiguity.
         */
        if (alg_id != TPM_ALG_SHA1) {
                for (i = TPM_PCR8; i < TPM_PCR10; i++) {
                        ima_pcrread(i, &d);
                        rc = crypto_shash_update(shash, d.digest,
                                                crypto_shash_digestsize(tfm));
                }
        }
        if (!rc)
                crypto_shash_final(shash, digest);
        return rc;
}

int ima_calc_boot_aggregate(struct ima_digest_data *hash)
{
        struct crypto_shash *tfm;
        u16 crypto_id, alg_id;
        int rc, i, bank_idx = -1;

        for (i = 0; i < ima_tpm_chip->nr_allocated_banks; i++) {
                crypto_id = ima_tpm_chip->allocated_banks[i].crypto_id;
                if (crypto_id == hash->algo) {
                        bank_idx = i;
                        break;
                }

                if (crypto_id == HASH_ALGO_SHA256)
                        bank_idx = i;

                if (bank_idx == -1 && crypto_id == HASH_ALGO_SHA1)
                        bank_idx = i;
        }

        if (bank_idx == -1) {
                pr_err("No suitable TPM algorithm for boot aggregate\n");
                return 0;
        }

        hash->algo = ima_tpm_chip->allocated_banks[bank_idx].crypto_id;

        tfm = ima_alloc_tfm(hash->algo);
        if (IS_ERR(tfm))
                return PTR_ERR(tfm);

        hash->length = crypto_shash_digestsize(tfm);
        alg_id = ima_tpm_chip->allocated_banks[bank_idx].alg_id;
        rc = ima_calc_boot_aggregate_tfm(hash->digest, alg_id, tfm);

        ima_free_tfm(tfm);

        return rc;
}




























































    7 








    8 









    2 




    2 



    1 




    1 



    2 




    2 



    2 




    2 








































    1 






































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (c) 2010-2013, The Linux Foundation. All rights reserved.
 */

#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/usb.h>
#include <linux/usb/ch11.h>

#define TEST_SE0_NAK_PID                        0x0101
#define TEST_J_PID                                0x0102
#define TEST_K_PID                                0x0103
#define TEST_PACKET_PID                                0x0104
#define TEST_HS_HOST_PORT_SUSPEND_RESUME        0x0106
#define TEST_SINGLE_STEP_GET_DEV_DESC                0x0107
#define TEST_SINGLE_STEP_SET_FEATURE                0x0108

extern const struct usb_device_id *usb_device_match_id(struct usb_device *udev,
                                                const struct usb_device_id *id);

/*
 * A list of USB hubs which requires to disable the power
 * to the port before starting the testing procedures.
 */
static const struct usb_device_id ehset_hub_list[] = {
        { USB_DEVICE(0x0424, 0x4502) },
        { USB_DEVICE(0x0424, 0x4913) },
        { USB_DEVICE(0x0451, 0x8027) },
        { }
};

static int ehset_prepare_port_for_testing(struct usb_device *hub_udev, u16 portnum)
{
        int ret = 0;

        /*
         * The USB2.0 spec chapter 11.24.2.13 says that the USB port which is
         * going under test needs to be put in suspend before sending the
         * test command. Most hubs don't enforce this precondition, but there
         * are some hubs which needs to disable the power to the port before
         * starting the test.
         */
        if (usb_device_match_id(hub_udev, ehset_hub_list)) {
                ret = usb_control_msg_send(hub_udev, 0, USB_REQ_CLEAR_FEATURE,
                                           USB_RT_PORT, USB_PORT_FEAT_ENABLE,
                                           portnum, NULL, 0, 1000, GFP_KERNEL);
                /*
                 * Wait for the port to be disabled. It's an arbitrary value
                 * which worked every time.
                 */
                msleep(100);
        } else {
                /*
                 * For the hubs which are compliant with the spec,
                 * put the port in SUSPEND.
                 */
                ret = usb_control_msg_send(hub_udev, 0, USB_REQ_SET_FEATURE,
                                           USB_RT_PORT, USB_PORT_FEAT_SUSPEND,
                                           portnum, NULL, 0, 1000, GFP_KERNEL);
        }
        return ret;
}

static int ehset_probe(struct usb_interface *intf,
                       const struct usb_device_id *id)
{
        int ret = -EINVAL;
        struct usb_device *dev = interface_to_usbdev(intf);
        struct usb_device *hub_udev = dev->parent;
        struct usb_device_descriptor buf;
        u8 portnum = dev->portnum;
        u16 test_pid = le16_to_cpu(dev->descriptor.idProduct);

        switch (test_pid) {
        case TEST_SE0_NAK_PID:
                ret = ehset_prepare_port_for_testing(hub_udev, portnum);
                if (ret < 0)
                        break;
                ret = usb_control_msg_send(hub_udev, 0, USB_REQ_SET_FEATURE,
                                           USB_RT_PORT, USB_PORT_FEAT_TEST,
                                           (USB_TEST_SE0_NAK << 8) | portnum,
                                           NULL, 0, 1000, GFP_KERNEL);
                break;
        case TEST_J_PID:
                ret = ehset_prepare_port_for_testing(hub_udev, portnum);
                if (ret < 0)
                        break;
                ret = usb_control_msg_send(hub_udev, 0, USB_REQ_SET_FEATURE,
                                           USB_RT_PORT, USB_PORT_FEAT_TEST,
                                           (USB_TEST_J << 8) | portnum, NULL, 0,
                                           1000, GFP_KERNEL);
                break;
        case TEST_K_PID:
                ret = ehset_prepare_port_for_testing(hub_udev, portnum);
                if (ret < 0)
                        break;
                ret = usb_control_msg_send(hub_udev, 0, USB_REQ_SET_FEATURE,
                                           USB_RT_PORT, USB_PORT_FEAT_TEST,
                                           (USB_TEST_K << 8) | portnum, NULL, 0,
                                           1000, GFP_KERNEL);
                break;
        case TEST_PACKET_PID:
                ret = ehset_prepare_port_for_testing(hub_udev, portnum);
                if (ret < 0)
                        break;
                ret = usb_control_msg_send(hub_udev, 0, USB_REQ_SET_FEATURE,
                                           USB_RT_PORT, USB_PORT_FEAT_TEST,
                                           (USB_TEST_PACKET << 8) | portnum,
                                           NULL, 0, 1000, GFP_KERNEL);
                break;
        case TEST_HS_HOST_PORT_SUSPEND_RESUME:
                /* Test: wait for 15secs -> suspend -> 15secs delay -> resume */
                msleep(15 * 1000);
                ret = usb_control_msg_send(hub_udev, 0, USB_REQ_SET_FEATURE,
                                           USB_RT_PORT, USB_PORT_FEAT_SUSPEND,
                                           portnum, NULL, 0, 1000, GFP_KERNEL);
                if (ret < 0)
                        break;

                msleep(15 * 1000);
                ret = usb_control_msg_send(hub_udev, 0, USB_REQ_CLEAR_FEATURE,
                                           USB_RT_PORT, USB_PORT_FEAT_SUSPEND,
                                           portnum, NULL, 0, 1000, GFP_KERNEL);
                break;
        case TEST_SINGLE_STEP_GET_DEV_DESC:
                /* Test: wait for 15secs -> GetDescriptor request */
                msleep(15 * 1000);

                ret = usb_control_msg_recv(dev, 0, USB_REQ_GET_DESCRIPTOR,
                                           USB_DIR_IN, USB_DT_DEVICE << 8, 0,
                                           &buf, USB_DT_DEVICE_SIZE,
                                           USB_CTRL_GET_TIMEOUT, GFP_KERNEL);
                break;
        case TEST_SINGLE_STEP_SET_FEATURE:
                /*
                 * GetDescriptor SETUP request -> 15secs delay -> IN & STATUS
                 *
                 * Note, this test is only supported on root hubs since the
                 * SetPortFeature handling can only be done inside the HCD's
                 * hub_control callback function.
                 */
                if (hub_udev != dev->bus->root_hub) {
                        dev_err(&intf->dev, "SINGLE_STEP_SET_FEATURE test only supported on root hub\n");
                        break;
                }

                ret = usb_control_msg_send(hub_udev, 0, USB_REQ_SET_FEATURE,
                                           USB_RT_PORT, USB_PORT_FEAT_TEST,
                                           (6 << 8) | portnum, NULL, 0,
                                           60 * 1000, GFP_KERNEL);

                break;
        default:
                dev_err(&intf->dev, "%s: unsupported PID: 0x%x\n",
                        __func__, test_pid);
        }

        return ret;
}

static void ehset_disconnect(struct usb_interface *intf)
{
}

static const struct usb_device_id ehset_id_table[] = {
        { USB_DEVICE(0x1a0a, TEST_SE0_NAK_PID) },
        { USB_DEVICE(0x1a0a, TEST_J_PID) },
        { USB_DEVICE(0x1a0a, TEST_K_PID) },
        { USB_DEVICE(0x1a0a, TEST_PACKET_PID) },
        { USB_DEVICE(0x1a0a, TEST_HS_HOST_PORT_SUSPEND_RESUME) },
        { USB_DEVICE(0x1a0a, TEST_SINGLE_STEP_GET_DEV_DESC) },
        { USB_DEVICE(0x1a0a, TEST_SINGLE_STEP_SET_FEATURE) },
        { }                        /* Terminating entry */
};
MODULE_DEVICE_TABLE(usb, ehset_id_table);

static struct usb_driver ehset_driver = {
        .name =                "usb_ehset_test",
        .probe =        ehset_probe,
        .disconnect =        ehset_disconnect,
        .id_table =        ehset_id_table,
};

module_usb_driver(ehset_driver);

MODULE_DESCRIPTION("USB Driver for EHSET Test Fixture");
MODULE_LICENSE("GPL v2");














































































































































































   10 























   12 






   12 














































   10 









   12 



































   11 

























   12 























   10 






































































































































1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
// SPDX-License-Identifier: GPL-2.0
/******************************************************************************
 * os_intfs.c
 *
 * Copyright(c) 2007 - 2010 Realtek Corporation. All rights reserved.
 * Linux device driver for RTL8192SU
 *
 * Modifications for inclusion into the Linux staging tree are
 * Copyright(c) 2010 Larry Finger. All rights reserved.
 *
 * Contact information:
 * WLAN FAE <wlanfae@realtek.com>.
 * Larry Finger <Larry.Finger@lwfinger.net>
 *
 ******************************************************************************/

#define _OS_INTFS_C_

#include <linux/module.h>
#include <linux/kthread.h>
#include <linux/firmware.h>
#include "osdep_service.h"
#include "drv_types.h"
#include "xmit_osdep.h"
#include "recv_osdep.h"
#include "rtl871x_ioctl.h"
#include "usb_osintf.h"

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("rtl871x wireless lan driver");
MODULE_AUTHOR("Larry Finger");

static char ifname[IFNAMSIZ] = "wlan%d";

/* module param defaults */
static int chip_version = RTL8712_2ndCUT;
static int rfintfs = HWPI;
static int lbkmode = RTL8712_AIR_TRX;
static int hci = RTL8712_USB;
static int ampdu_enable = 1;/*for enable tx_ampdu*/

/* The video_mode variable is for video mode.*/
/* It may be specify when inserting module with video_mode=1 parameter.*/
static int video_mode = 1;   /* enable video mode*/

/*Ndis802_11Infrastructure; infra, ad-hoc, auto*/
static int network_mode = Ndis802_11IBSS;
static int channel = 1;/*ad-hoc support requirement*/
static int wireless_mode = WIRELESS_11BG;
static int vrtl_carrier_sense = AUTO_VCS;
static int vcs_type = RTS_CTS;
static int frag_thresh = 2346;
static int preamble = PREAMBLE_LONG;/*long, short, auto*/
static int scan_mode = 1;/*active, passive*/
static int adhoc_tx_pwr = 1;
static int soft_ap;
static int smart_ps = 1;
static int power_mgnt = PS_MODE_ACTIVE;
static int radio_enable = 1;
static int long_retry_lmt = 7;
static int short_retry_lmt = 7;
static int busy_thresh = 40;
static int ack_policy = NORMAL_ACK;
static int mp_mode;
static int software_encrypt;
static int software_decrypt;

static int wmm_enable;/* default is set to disable the wmm.*/
static int uapsd_enable;
static int uapsd_max_sp = NO_LIMIT;
static int uapsd_acbk_en;
static int uapsd_acbe_en;
static int uapsd_acvi_en;
static int uapsd_acvo_en;

static int ht_enable = 1;
static int cbw40_enable = 1;
static int rf_config = RTL8712_RF_1T2R;  /* 1T2R*/
static int low_power;
/* mac address to use instead of the one stored in Efuse */
char *r8712_initmac;
static char *initmac;
/* if wifi_test = 1, driver will disable the turbo mode and pass it to
 * firmware private.
 */
static int wifi_test;

module_param_string(ifname, ifname, sizeof(ifname), 0644);
module_param(wifi_test, int, 0644);
module_param(initmac, charp, 0644);
module_param(video_mode, int, 0644);
module_param(chip_version, int, 0644);
module_param(rfintfs, int, 0644);
module_param(lbkmode, int, 0644);
module_param(hci, int, 0644);
module_param(network_mode, int, 0644);
module_param(channel, int, 0644);
module_param(mp_mode, int, 0644);
module_param(wmm_enable, int, 0644);
module_param(vrtl_carrier_sense, int, 0644);
module_param(vcs_type, int, 0644);
module_param(busy_thresh, int, 0644);
module_param(ht_enable, int, 0644);
module_param(cbw40_enable, int, 0644);
module_param(ampdu_enable, int, 0644);
module_param(rf_config, int, 0644);
module_param(power_mgnt, int, 0644);
module_param(low_power, int, 0644);

MODULE_PARM_DESC(ifname, " Net interface name, wlan%d=default");
MODULE_PARM_DESC(initmac, "MAC-Address, default: use FUSE");

static int netdev_open(struct net_device *pnetdev);
static int netdev_close(struct net_device *pnetdev);

static void loadparam(struct _adapter *padapter, struct  net_device *pnetdev)
{
        struct registry_priv  *registry_par = &padapter->registrypriv;

        registry_par->chip_version = (u8)chip_version;
        registry_par->rfintfs = (u8)rfintfs;
        registry_par->lbkmode = (u8)lbkmode;
        registry_par->hci = (u8)hci;
        registry_par->network_mode  = (u8)network_mode;
        memcpy(registry_par->ssid.Ssid, "ANY", 3);
        registry_par->ssid.SsidLength = 3;
        registry_par->channel = (u8)channel;
        registry_par->wireless_mode = (u8)wireless_mode;
        registry_par->vrtl_carrier_sense = (u8)vrtl_carrier_sense;
        registry_par->vcs_type = (u8)vcs_type;
        registry_par->frag_thresh = (u16)frag_thresh;
        registry_par->preamble = (u8)preamble;
        registry_par->scan_mode = (u8)scan_mode;
        registry_par->adhoc_tx_pwr = (u8)adhoc_tx_pwr;
        registry_par->soft_ap = (u8)soft_ap;
        registry_par->smart_ps = (u8)smart_ps;
        registry_par->power_mgnt = (u8)power_mgnt;
        registry_par->radio_enable = (u8)radio_enable;
        registry_par->long_retry_lmt = (u8)long_retry_lmt;
        registry_par->short_retry_lmt = (u8)short_retry_lmt;
        registry_par->busy_thresh = (u16)busy_thresh;
        registry_par->ack_policy = (u8)ack_policy;
        registry_par->mp_mode = (u8)mp_mode;
        registry_par->software_encrypt = (u8)software_encrypt;
        registry_par->software_decrypt = (u8)software_decrypt;
        /*UAPSD*/
        registry_par->wmm_enable = (u8)wmm_enable;
        registry_par->uapsd_enable = (u8)uapsd_enable;
        registry_par->uapsd_max_sp = (u8)uapsd_max_sp;
        registry_par->uapsd_acbk_en = (u8)uapsd_acbk_en;
        registry_par->uapsd_acbe_en = (u8)uapsd_acbe_en;
        registry_par->uapsd_acvi_en = (u8)uapsd_acvi_en;
        registry_par->uapsd_acvo_en = (u8)uapsd_acvo_en;
        registry_par->ht_enable = (u8)ht_enable;
        registry_par->cbw40_enable = (u8)cbw40_enable;
        registry_par->ampdu_enable = (u8)ampdu_enable;
        registry_par->rf_config = (u8)rf_config;
        registry_par->low_power = (u8)low_power;
        registry_par->wifi_test = (u8)wifi_test;
        r8712_initmac = initmac;
}

static int r871x_net_set_mac_address(struct net_device *pnetdev, void *p)
{
        struct _adapter *padapter = netdev_priv(pnetdev);
        struct sockaddr *addr = p;

        if (!padapter->bup)
                eth_hw_addr_set(pnetdev, addr->sa_data);
        return 0;
}

static struct net_device_stats *r871x_net_get_stats(struct net_device *pnetdev)
{
        struct _adapter *padapter = netdev_priv(pnetdev);
        struct xmit_priv *pxmitpriv = &padapter->xmitpriv;
        struct recv_priv *precvpriv = &padapter->recvpriv;

        padapter->stats.tx_packets = pxmitpriv->tx_pkts;
        padapter->stats.rx_packets = precvpriv->rx_pkts;
        padapter->stats.tx_dropped = pxmitpriv->tx_drop;
        padapter->stats.rx_dropped = precvpriv->rx_drop;
        padapter->stats.tx_bytes = pxmitpriv->tx_bytes;
        padapter->stats.rx_bytes = precvpriv->rx_bytes;
        return &padapter->stats;
}

static const struct net_device_ops rtl8712_netdev_ops = {
        .ndo_open = netdev_open,
        .ndo_stop = netdev_close,
        .ndo_start_xmit = r8712_xmit_entry,
        .ndo_set_mac_address = r871x_net_set_mac_address,
        .ndo_get_stats = r871x_net_get_stats,
        .ndo_do_ioctl = r871x_ioctl,
};

struct net_device *r8712_init_netdev(void)
{
        struct _adapter *padapter;
        struct net_device *pnetdev;

        pnetdev = alloc_etherdev(sizeof(struct _adapter));
        if (!pnetdev)
                return NULL;
        if (dev_alloc_name(pnetdev, ifname) < 0) {
                strscpy(ifname, "wlan%d", sizeof(ifname));
                dev_alloc_name(pnetdev, ifname);
        }
        padapter = netdev_priv(pnetdev);
        padapter->pnetdev = pnetdev;
        pr_info("r8712u: register rtl8712_netdev_ops to netdev_ops\n");
        pnetdev->netdev_ops = &rtl8712_netdev_ops;
        pnetdev->watchdog_timeo = HZ; /* 1 second timeout */
        pnetdev->wireless_handlers = (struct iw_handler_def *)
                                     &r871x_handlers_def;
        loadparam(padapter, pnetdev);
        netif_carrier_off(pnetdev);
        padapter->pid = 0;  /* Initial the PID value used for HW PBC.*/
        return pnetdev;
}

static u32 start_drv_threads(struct _adapter *padapter)
{
        padapter->cmd_thread = kthread_run(r8712_cmd_thread, padapter, "%s",
                                           padapter->pnetdev->name);
        if (IS_ERR(padapter->cmd_thr