| 113 113 30 139 64 38 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _LINUX_CPUSET_H #define _LINUX_CPUSET_H /* * cpuset interface * * Copyright (C) 2003 BULL SA * Copyright (C) 2004-2006 Silicon Graphics, Inc. * */ #include <linux/sched.h> #include <linux/sched/topology.h> #include <linux/sched/task.h> #include <linux/cpumask.h> #include <linux/nodemask.h> #include <linux/mm.h> #include <linux/mmu_context.h> #include <linux/jump_label.h> #ifdef CONFIG_CPUSETS /* * Static branch rewrites can happen in an arbitrary order for a given * key. In code paths where we need to loop with read_mems_allowed_begin() and * read_mems_allowed_retry() to get a consistent view of mems_allowed, we need * to ensure that begin() always gets rewritten before retry() in the * disabled -> enabled transition. If not, then if local irqs are disabled * around the loop, we can deadlock since retry() would always be * comparing the latest value of the mems_allowed seqcount against 0 as * begin() still would see cpusets_enabled() as false. The enabled -> disabled * transition should happen in reverse order for the same reasons (want to stop * looking at real value of mems_allowed.sequence in retry() first). */ extern struct static_key_false cpusets_pre_enable_key; extern struct static_key_false cpusets_enabled_key; extern struct static_key_false cpusets_insane_config_key; static inline bool cpusets_enabled(void) { return static_branch_unlikely(&cpusets_enabled_key); } static inline void cpuset_inc(void) { static_branch_inc_cpuslocked(&cpusets_pre_enable_key); static_branch_inc_cpuslocked(&cpusets_enabled_key); } static inline void cpuset_dec(void) { static_branch_dec_cpuslocked(&cpusets_enabled_key); static_branch_dec_cpuslocked(&cpusets_pre_enable_key); } /* * This will get enabled whenever a cpuset configuration is considered * unsupportable in general. E.g. movable only node which cannot satisfy * any non movable allocations (see update_nodemask). Page allocator * needs to make additional checks for those configurations and this * check is meant to guard those checks without any overhead for sane * configurations. */ static inline bool cpusets_insane_config(void) { return static_branch_unlikely(&cpusets_insane_config_key); } extern int cpuset_init(void); extern void cpuset_init_smp(void); extern void cpuset_force_rebuild(void); extern void cpuset_update_active_cpus(void); extern void inc_dl_tasks_cs(struct task_struct *task); extern void dec_dl_tasks_cs(struct task_struct *task); extern void cpuset_lock(void); extern void cpuset_unlock(void); extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); extern bool cpuset_cpu_is_isolated(int cpu); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask); extern bool cpuset_current_node_allowed(int node, gfp_t gfp_mask); static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) { return cpuset_current_node_allowed(zone_to_nid(z), gfp_mask); } static inline bool cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) { if (cpusets_enabled()) return __cpuset_zone_allowed(z, gfp_mask); return true; } extern int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, const struct task_struct *tsk2); #ifdef CONFIG_CPUSETS_V1 #define cpuset_memory_pressure_bump() \ do { \ if (cpuset_memory_pressure_enabled) \ __cpuset_memory_pressure_bump(); \ } while (0) extern int cpuset_memory_pressure_enabled; extern void __cpuset_memory_pressure_bump(void); #else static inline void cpuset_memory_pressure_bump(void) { } #endif extern void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task); extern int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *tsk); extern int cpuset_mem_spread_node(void); static inline int cpuset_do_page_mem_spread(void) { return task_spread_page(current); } extern bool current_cpuset_is_being_rebound(void); extern void dl_rebuild_rd_accounting(void); extern void rebuild_sched_domains(void); extern void cpuset_print_current_mems_allowed(void); extern void cpuset_reset_sched_domains(void); /* * read_mems_allowed_begin is required when making decisions involving * mems_allowed such as during page allocation. mems_allowed can be updated in * parallel and depending on the new value an operation can fail potentially * causing process failure. A retry loop with read_mems_allowed_begin and * read_mems_allowed_retry prevents these artificial failures. */ static inline unsigned int read_mems_allowed_begin(void) { if (!static_branch_unlikely(&cpusets_pre_enable_key)) return 0; return read_seqcount_begin(¤t->mems_allowed_seq); } /* * If this returns true, the operation that took place after * read_mems_allowed_begin may have failed artificially due to a concurrent * update of mems_allowed. It is up to the caller to retry the operation if * appropriate. */ static inline bool read_mems_allowed_retry(unsigned int seq) { if (!static_branch_unlikely(&cpusets_enabled_key)) return false; return read_seqcount_retry(¤t->mems_allowed_seq, seq); } static inline void set_mems_allowed(nodemask_t nodemask) { unsigned long flags; task_lock(current); local_irq_save(flags); write_seqcount_begin(¤t->mems_allowed_seq); current->mems_allowed = nodemask; write_seqcount_end(¤t->mems_allowed_seq); local_irq_restore(flags); task_unlock(current); } extern bool cpuset_node_allowed(struct cgroup *cgroup, int nid); #else /* !CONFIG_CPUSETS */ static inline bool cpusets_enabled(void) { return false; } static inline bool cpusets_insane_config(void) { return false; } static inline int cpuset_init(void) { return 0; } static inline void cpuset_init_smp(void) {} static inline void cpuset_force_rebuild(void) { } static inline void cpuset_update_active_cpus(void) { partition_sched_domains(1, NULL, NULL); } static inline void inc_dl_tasks_cs(struct task_struct *task) { } static inline void dec_dl_tasks_cs(struct task_struct *task) { } static inline void cpuset_lock(void) { } static inline void cpuset_unlock(void) { } static inline void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask) { cpumask_copy(mask, task_cpu_possible_mask(p)); } static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p) { return false; } static inline bool cpuset_cpu_is_isolated(int cpu) { return false; } static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) { return node_possible_map; } #define cpuset_current_mems_allowed (node_states[N_MEMORY]) static inline void cpuset_init_current_mems_allowed(void) {} static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) { return 1; } static inline bool __cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) { return true; } static inline bool cpuset_zone_allowed(struct zone *z, gfp_t gfp_mask) { return true; } static inline int cpuset_mems_allowed_intersects(const struct task_struct *tsk1, const struct task_struct *tsk2) { return 1; } static inline void cpuset_memory_pressure_bump(void) {} static inline void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task) { } static inline int cpuset_mem_spread_node(void) { return 0; } static inline int cpuset_do_page_mem_spread(void) { return 0; } static inline bool current_cpuset_is_being_rebound(void) { return false; } static inline void dl_rebuild_rd_accounting(void) { } static inline void rebuild_sched_domains(void) { partition_sched_domains(1, NULL, NULL); } static inline void cpuset_reset_sched_domains(void) { partition_sched_domains(1, NULL, NULL); } static inline void cpuset_print_current_mems_allowed(void) { } static inline void set_mems_allowed(nodemask_t nodemask) { } static inline unsigned int read_mems_allowed_begin(void) { return 0; } static inline bool read_mems_allowed_retry(unsigned int seq) { return false; } static inline bool cpuset_node_allowed(struct cgroup *cgroup, int nid) { return true; } #endif /* !CONFIG_CPUSETS */ #endif /* _LINUX_CPUSET_H */ |
| 4 4 4 5 1 4 52 52 28 28 27 25 1 1 26 2 13 1 24 49 20 2 27 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 | // SPDX-License-Identifier: GPL-2.0 /* * Functions related to io context handling */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/slab.h> #include <linux/security.h> #include <linux/sched/task.h> #include "blk.h" #include "blk-mq-sched.h" /* * For io context allocations */ static struct kmem_cache *iocontext_cachep; #ifdef CONFIG_BLK_ICQ /** * get_io_context - increment reference count to io_context * @ioc: io_context to get * * Increment reference count to @ioc. */ static void get_io_context(struct io_context *ioc) { BUG_ON(atomic_long_read(&ioc->refcount) <= 0); atomic_long_inc(&ioc->refcount); } /* * Exit an icq. Called with ioc locked for blk-mq, and with both ioc * and queue locked for legacy. */ static void ioc_exit_icq(struct io_cq *icq) { struct elevator_type *et = icq->q->elevator->type; if (icq->flags & ICQ_EXITED) return; if (et->ops.exit_icq) et->ops.exit_icq(icq); icq->flags |= ICQ_EXITED; } static void ioc_exit_icqs(struct io_context *ioc) { struct io_cq *icq; spin_lock_irq(&ioc->lock); hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) ioc_exit_icq(icq); spin_unlock_irq(&ioc->lock); } /* * Release an icq. Called with ioc locked for blk-mq, and with both ioc * and queue locked for legacy. */ static void ioc_destroy_icq(struct io_cq *icq) { struct io_context *ioc = icq->ioc; struct request_queue *q = icq->q; struct elevator_type *et = q->elevator->type; lockdep_assert_held(&ioc->lock); lockdep_assert_held(&q->queue_lock); if (icq->flags & ICQ_DESTROYED) return; radix_tree_delete(&ioc->icq_tree, icq->q->id); hlist_del_init(&icq->ioc_node); list_del_init(&icq->q_node); /* * Both setting lookup hint to and clearing it from @icq are done * under queue_lock. If it's not pointing to @icq now, it never * will. Hint assignment itself can race safely. */ if (rcu_access_pointer(ioc->icq_hint) == icq) rcu_assign_pointer(ioc->icq_hint, NULL); ioc_exit_icq(icq); /* * @icq->q might have gone away by the time RCU callback runs * making it impossible to determine icq_cache. Record it in @icq. */ icq->__rcu_icq_cache = et->icq_cache; icq->flags |= ICQ_DESTROYED; kfree_rcu(icq, __rcu_head); } /* * Slow path for ioc release in put_io_context(). Performs double-lock * dancing to unlink all icq's and then frees ioc. */ static void ioc_release_fn(struct work_struct *work) { struct io_context *ioc = container_of(work, struct io_context, release_work); spin_lock_irq(&ioc->lock); while (!hlist_empty(&ioc->icq_list)) { struct io_cq *icq = hlist_entry(ioc->icq_list.first, struct io_cq, ioc_node); struct request_queue *q = icq->q; if (spin_trylock(&q->queue_lock)) { ioc_destroy_icq(icq); spin_unlock(&q->queue_lock); } else { /* Make sure q and icq cannot be freed. */ rcu_read_lock(); /* Re-acquire the locks in the correct order. */ spin_unlock(&ioc->lock); spin_lock(&q->queue_lock); spin_lock(&ioc->lock); ioc_destroy_icq(icq); spin_unlock(&q->queue_lock); rcu_read_unlock(); } } spin_unlock_irq(&ioc->lock); kmem_cache_free(iocontext_cachep, ioc); } /* * Releasing icqs requires reverse order double locking and we may already be * holding a queue_lock. Do it asynchronously from a workqueue. */ static bool ioc_delay_free(struct io_context *ioc) { unsigned long flags; spin_lock_irqsave(&ioc->lock, flags); if (!hlist_empty(&ioc->icq_list)) { queue_work(system_power_efficient_wq, &ioc->release_work); spin_unlock_irqrestore(&ioc->lock, flags); return true; } spin_unlock_irqrestore(&ioc->lock, flags); return false; } /** * ioc_clear_queue - break any ioc association with the specified queue * @q: request_queue being cleared * * Walk @q->icq_list and exit all io_cq's. */ void ioc_clear_queue(struct request_queue *q) { spin_lock_irq(&q->queue_lock); while (!list_empty(&q->icq_list)) { struct io_cq *icq = list_first_entry(&q->icq_list, struct io_cq, q_node); /* * Other context won't hold ioc lock to wait for queue_lock, see * details in ioc_release_fn(). */ spin_lock(&icq->ioc->lock); ioc_destroy_icq(icq); spin_unlock(&icq->ioc->lock); } spin_unlock_irq(&q->queue_lock); } #else /* CONFIG_BLK_ICQ */ static inline void ioc_exit_icqs(struct io_context *ioc) { } static inline bool ioc_delay_free(struct io_context *ioc) { return false; } #endif /* CONFIG_BLK_ICQ */ /** * put_io_context - put a reference of io_context * @ioc: io_context to put * * Decrement reference count of @ioc and release it if the count reaches * zero. */ void put_io_context(struct io_context *ioc) { BUG_ON(atomic_long_read(&ioc->refcount) <= 0); if (atomic_long_dec_and_test(&ioc->refcount) && !ioc_delay_free(ioc)) kmem_cache_free(iocontext_cachep, ioc); } EXPORT_SYMBOL_GPL(put_io_context); /* Called by the exiting task */ void exit_io_context(struct task_struct *task) { struct io_context *ioc; task_lock(task); ioc = task->io_context; task->io_context = NULL; task_unlock(task); if (atomic_dec_and_test(&ioc->active_ref)) { ioc_exit_icqs(ioc); put_io_context(ioc); } } static struct io_context *alloc_io_context(gfp_t gfp_flags, int node) { struct io_context *ioc; ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO, node); if (unlikely(!ioc)) return NULL; atomic_long_set(&ioc->refcount, 1); atomic_set(&ioc->active_ref, 1); #ifdef CONFIG_BLK_ICQ spin_lock_init(&ioc->lock); INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC); INIT_HLIST_HEAD(&ioc->icq_list); INIT_WORK(&ioc->release_work, ioc_release_fn); #endif ioc->ioprio = IOPRIO_DEFAULT; return ioc; } int set_task_ioprio(struct task_struct *task, int ioprio) { int err; const struct cred *cred = current_cred(), *tcred; rcu_read_lock(); tcred = __task_cred(task); if (!uid_eq(tcred->uid, cred->euid) && !uid_eq(tcred->uid, cred->uid) && !capable(CAP_SYS_NICE)) { rcu_read_unlock(); return -EPERM; } rcu_read_unlock(); err = security_task_setioprio(task, ioprio); if (err) return err; task_lock(task); if (unlikely(!task->io_context)) { struct io_context *ioc; task_unlock(task); ioc = alloc_io_context(GFP_ATOMIC, NUMA_NO_NODE); if (!ioc) return -ENOMEM; task_lock(task); if (task->flags & PF_EXITING) { kmem_cache_free(iocontext_cachep, ioc); goto out; } if (task->io_context) kmem_cache_free(iocontext_cachep, ioc); else task->io_context = ioc; } task->io_context->ioprio = ioprio; out: task_unlock(task); return 0; } EXPORT_SYMBOL_GPL(set_task_ioprio); int __copy_io(u64 clone_flags, struct task_struct *tsk) { struct io_context *ioc = current->io_context; /* * Share io context with parent, if CLONE_IO is set */ if (clone_flags & CLONE_IO) { atomic_inc(&ioc->active_ref); tsk->io_context = ioc; } else if (ioprio_valid(ioc->ioprio)) { tsk->io_context = alloc_io_context(GFP_KERNEL, NUMA_NO_NODE); if (!tsk->io_context) return -ENOMEM; tsk->io_context->ioprio = ioc->ioprio; } return 0; } #ifdef CONFIG_BLK_ICQ /** * ioc_lookup_icq - lookup io_cq from ioc in io issue path * @q: the associated request_queue * * Look up io_cq associated with @ioc - @q pair from @ioc. Must be called * from io issue path, either return NULL if current issue io to @q for the * first time, or return a valid icq. */ struct io_cq *ioc_lookup_icq(struct request_queue *q) { struct io_context *ioc = current->io_context; struct io_cq *icq; /* * icq's are indexed from @ioc using radix tree and hint pointer, * both of which are protected with RCU, io issue path ensures that * both request_queue and current task are valid, the found icq * is guaranteed to be valid until the io is done. */ rcu_read_lock(); icq = rcu_dereference(ioc->icq_hint); if (icq && icq->q == q) goto out; icq = radix_tree_lookup(&ioc->icq_tree, q->id); if (icq && icq->q == q) rcu_assign_pointer(ioc->icq_hint, icq); /* allowed to race */ else icq = NULL; out: rcu_read_unlock(); return icq; } EXPORT_SYMBOL(ioc_lookup_icq); /** * ioc_create_icq - create and link io_cq * @q: request_queue of interest * * Make sure io_cq linking @ioc and @q exists. If icq doesn't exist, they * will be created using @gfp_mask. * * The caller is responsible for ensuring @ioc won't go away and @q is * alive and will stay alive until this function returns. */ static struct io_cq *ioc_create_icq(struct request_queue *q) { struct io_context *ioc = current->io_context; struct elevator_type *et = q->elevator->type; struct io_cq *icq; /* allocate stuff */ icq = kmem_cache_alloc_node(et->icq_cache, GFP_ATOMIC | __GFP_ZERO, q->node); if (!icq) return NULL; if (radix_tree_maybe_preload(GFP_ATOMIC) < 0) { kmem_cache_free(et->icq_cache, icq); return NULL; } icq->ioc = ioc; icq->q = q; INIT_LIST_HEAD(&icq->q_node); INIT_HLIST_NODE(&icq->ioc_node); /* lock both q and ioc and try to link @icq */ spin_lock_irq(&q->queue_lock); spin_lock(&ioc->lock); if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) { hlist_add_head(&icq->ioc_node, &ioc->icq_list); list_add(&icq->q_node, &q->icq_list); if (et->ops.init_icq) et->ops.init_icq(icq); } else { kmem_cache_free(et->icq_cache, icq); icq = ioc_lookup_icq(q); if (!icq) printk(KERN_ERR "cfq: icq link failed!\n"); } spin_unlock(&ioc->lock); spin_unlock_irq(&q->queue_lock); radix_tree_preload_end(); return icq; } struct io_cq *ioc_find_get_icq(struct request_queue *q) { struct io_context *ioc = current->io_context; struct io_cq *icq = NULL; if (unlikely(!ioc)) { ioc = alloc_io_context(GFP_ATOMIC, q->node); if (!ioc) return NULL; task_lock(current); if (current->io_context) { kmem_cache_free(iocontext_cachep, ioc); ioc = current->io_context; } else { current->io_context = ioc; } get_io_context(ioc); task_unlock(current); } else { get_io_context(ioc); icq = ioc_lookup_icq(q); } if (!icq) { icq = ioc_create_icq(q); if (!icq) { put_io_context(ioc); return NULL; } } return icq; } EXPORT_SYMBOL_GPL(ioc_find_get_icq); #endif /* CONFIG_BLK_ICQ */ static int __init blk_ioc_init(void) { iocontext_cachep = kmem_cache_create("blkdev_ioc", sizeof(struct io_context), 0, SLAB_PANIC, NULL); return 0; } subsys_initcall(blk_ioc_init); |
| 1 1 4 6 6 6 2 4 2 2 2 4 4 1 1 2 4 1 1 1 1 1 1 1 1 1 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 | // SPDX-License-Identifier: GPL-2.0-only /* * * Copyright (C) Hans Alblas PE1AYX <hans@esrac.ele.tue.nl> * Copyright (C) 2004, 05 Ralf Baechle DL5RB <ralf@linux-mips.org> * Copyright (C) 2004, 05 Thomas Osterried DL9SAU <thomas@x-berg.in-berlin.de> */ #include <linux/module.h> #include <linux/bitops.h> #include <linux/uaccess.h> #include <linux/crc16.h> #include <linux/string.h> #include <linux/mm.h> #include <linux/interrupt.h> #include <linux/in.h> #include <linux/inet.h> #include <linux/slab.h> #include <linux/tty.h> #include <linux/errno.h> #include <linux/netdevice.h> #include <linux/major.h> #include <linux/init.h> #include <linux/rtnetlink.h> #include <linux/etherdevice.h> #include <linux/skbuff.h> #include <linux/if_arp.h> #include <linux/jiffies.h> #include <linux/refcount.h> #include <net/ax25.h> #define AX_MTU 236 /* some arch define END as assembly function ending, just undef it */ #undef END /* SLIP/KISS protocol characters. */ #define END 0300 /* indicates end of frame */ #define ESC 0333 /* indicates byte stuffing */ #define ESC_END 0334 /* ESC ESC_END means END 'data' */ #define ESC_ESC 0335 /* ESC ESC_ESC means ESC 'data' */ struct mkiss { struct tty_struct *tty; /* ptr to TTY structure */ struct net_device *dev; /* easy for intr handling */ /* These are pointers to the malloc()ed frame buffers. */ spinlock_t buflock;/* lock for rbuf and xbuf */ unsigned char *rbuff; /* receiver buffer */ int rcount; /* received chars counter */ unsigned char *xbuff; /* transmitter buffer */ unsigned char *xhead; /* pointer to next byte to XMIT */ int xleft; /* bytes left in XMIT queue */ /* Detailed SLIP statistics. */ int mtu; /* Our mtu (to spot changes!) */ int buffsize; /* Max buffers sizes */ unsigned long flags; /* Flag values/ mode etc */ /* long req'd: used by set_bit --RR */ #define AXF_INUSE 0 /* Channel in use */ #define AXF_ESCAPE 1 /* ESC received */ #define AXF_ERROR 2 /* Parity, etc. error */ #define AXF_KEEPTEST 3 /* Keepalive test flag */ #define AXF_OUTWAIT 4 /* is outpacket was flag */ int mode; int crcmode; /* MW: for FlexNet, SMACK etc. */ int crcauto; /* CRC auto mode */ #define CRC_MODE_NONE 0 #define CRC_MODE_FLEX 1 #define CRC_MODE_SMACK 2 #define CRC_MODE_FLEX_TEST 3 #define CRC_MODE_SMACK_TEST 4 refcount_t refcnt; struct completion dead; }; /*---------------------------------------------------------------------------*/ static const unsigned short crc_flex_table[] = { 0x0f87, 0x1e0e, 0x2c95, 0x3d1c, 0x49a3, 0x582a, 0x6ab1, 0x7b38, 0x83cf, 0x9246, 0xa0dd, 0xb154, 0xc5eb, 0xd462, 0xe6f9, 0xf770, 0x1f06, 0x0e8f, 0x3c14, 0x2d9d, 0x5922, 0x48ab, 0x7a30, 0x6bb9, 0x934e, 0x82c7, 0xb05c, 0xa1d5, 0xd56a, 0xc4e3, 0xf678, 0xe7f1, 0x2e85, 0x3f0c, 0x0d97, 0x1c1e, 0x68a1, 0x7928, 0x4bb3, 0x5a3a, 0xa2cd, 0xb344, 0x81df, 0x9056, 0xe4e9, 0xf560, 0xc7fb, 0xd672, 0x3e04, 0x2f8d, 0x1d16, 0x0c9f, 0x7820, 0x69a9, 0x5b32, 0x4abb, 0xb24c, 0xa3c5, 0x915e, 0x80d7, 0xf468, 0xe5e1, 0xd77a, 0xc6f3, 0x4d83, 0x5c0a, 0x6e91, 0x7f18, 0x0ba7, 0x1a2e, 0x28b5, 0x393c, 0xc1cb, 0xd042, 0xe2d9, 0xf350, 0x87ef, 0x9666, 0xa4fd, 0xb574, 0x5d02, 0x4c8b, 0x7e10, 0x6f99, 0x1b26, 0x0aaf, 0x3834, 0x29bd, 0xd14a, 0xc0c3, 0xf258, 0xe3d1, 0x976e, 0x86e7, 0xb47c, 0xa5f5, 0x6c81, 0x7d08, 0x4f93, 0x5e1a, 0x2aa5, 0x3b2c, 0x09b7, 0x183e, 0xe0c9, 0xf140, 0xc3db, 0xd252, 0xa6ed, 0xb764, 0x85ff, 0x9476, 0x7c00, 0x6d89, 0x5f12, 0x4e9b, 0x3a24, 0x2bad, 0x1936, 0x08bf, 0xf048, 0xe1c1, 0xd35a, 0xc2d3, 0xb66c, 0xa7e5, 0x957e, 0x84f7, 0x8b8f, 0x9a06, 0xa89d, 0xb914, 0xcdab, 0xdc22, 0xeeb9, 0xff30, 0x07c7, 0x164e, 0x24d5, 0x355c, 0x41e3, 0x506a, 0x62f1, 0x7378, 0x9b0e, 0x8a87, 0xb81c, 0xa995, 0xdd2a, 0xcca3, 0xfe38, 0xefb1, 0x1746, 0x06cf, 0x3454, 0x25dd, 0x5162, 0x40eb, 0x7270, 0x63f9, 0xaa8d, 0xbb04, 0x899f, 0x9816, 0xeca9, 0xfd20, 0xcfbb, 0xde32, 0x26c5, 0x374c, 0x05d7, 0x145e, 0x60e1, 0x7168, 0x43f3, 0x527a, 0xba0c, 0xab85, 0x991e, 0x8897, 0xfc28, 0xeda1, 0xdf3a, 0xceb3, 0x3644, 0x27cd, 0x1556, 0x04df, 0x7060, 0x61e9, 0x5372, 0x42fb, 0xc98b, 0xd802, 0xea99, 0xfb10, 0x8faf, 0x9e26, 0xacbd, 0xbd34, 0x45c3, 0x544a, 0x66d1, 0x7758, 0x03e7, 0x126e, 0x20f5, 0x317c, 0xd90a, 0xc883, 0xfa18, 0xeb91, 0x9f2e, 0x8ea7, 0xbc3c, 0xadb5, 0x5542, 0x44cb, 0x7650, 0x67d9, 0x1366, 0x02ef, 0x3074, 0x21fd, 0xe889, 0xf900, 0xcb9b, 0xda12, 0xaead, 0xbf24, 0x8dbf, 0x9c36, 0x64c1, 0x7548, 0x47d3, 0x565a, 0x22e5, 0x336c, 0x01f7, 0x107e, 0xf808, 0xe981, 0xdb1a, 0xca93, 0xbe2c, 0xafa5, 0x9d3e, 0x8cb7, 0x7440, 0x65c9, 0x5752, 0x46db, 0x3264, 0x23ed, 0x1176, 0x00ff }; static unsigned short calc_crc_flex(unsigned char *cp, int size) { unsigned short crc = 0xffff; while (size--) crc = (crc << 8) ^ crc_flex_table[((crc >> 8) ^ *cp++) & 0xff]; return crc; } static int check_crc_flex(unsigned char *cp, int size) { unsigned short crc = 0xffff; if (size < 3) return -1; while (size--) crc = (crc << 8) ^ crc_flex_table[((crc >> 8) ^ *cp++) & 0xff]; if ((crc & 0xffff) != 0x7070) return -1; return 0; } static int check_crc_16(unsigned char *cp, int size) { unsigned short crc = 0x0000; if (size < 3) return -1; crc = crc16(0, cp, size); if (crc != 0x0000) return -1; return 0; } /* * Standard encapsulation */ static int kiss_esc(unsigned char *s, unsigned char *d, int len) { unsigned char *ptr = d; unsigned char c; /* * Send an initial END character to flush out any data that may have * accumulated in the receiver due to line noise. */ *ptr++ = END; while (len-- > 0) { switch (c = *s++) { case END: *ptr++ = ESC; *ptr++ = ESC_END; break; case ESC: *ptr++ = ESC; *ptr++ = ESC_ESC; break; default: *ptr++ = c; break; } } *ptr++ = END; return ptr - d; } /* * MW: * OK its ugly, but tell me a better solution without copying the * packet to a temporary buffer :-) */ static int kiss_esc_crc(unsigned char *s, unsigned char *d, unsigned short crc, int len) { unsigned char *ptr = d; unsigned char c=0; *ptr++ = END; while (len > 0) { if (len > 2) c = *s++; else if (len > 1) c = crc >> 8; else c = crc & 0xff; len--; switch (c) { case END: *ptr++ = ESC; *ptr++ = ESC_END; break; case ESC: *ptr++ = ESC; *ptr++ = ESC_ESC; break; default: *ptr++ = c; break; } } *ptr++ = END; return ptr - d; } /* Send one completely decapsulated AX.25 packet to the AX.25 layer. */ static void ax_bump(struct mkiss *ax) { struct sk_buff *skb; int count; spin_lock_bh(&ax->buflock); if (ax->rbuff[0] > 0x0f) { if (ax->rbuff[0] & 0x80) { if (check_crc_16(ax->rbuff, ax->rcount) < 0) { ax->dev->stats.rx_errors++; spin_unlock_bh(&ax->buflock); return; } if (ax->crcmode != CRC_MODE_SMACK && ax->crcauto) { printk(KERN_INFO "mkiss: %s: Switching to crc-smack\n", ax->dev->name); ax->crcmode = CRC_MODE_SMACK; } ax->rcount -= 2; *ax->rbuff &= ~0x80; } else if (ax->rbuff[0] & 0x20) { if (check_crc_flex(ax->rbuff, ax->rcount) < 0) { ax->dev->stats.rx_errors++; spin_unlock_bh(&ax->buflock); return; } if (ax->crcmode != CRC_MODE_FLEX && ax->crcauto) { printk(KERN_INFO "mkiss: %s: Switching to crc-flexnet\n", ax->dev->name); ax->crcmode = CRC_MODE_FLEX; } ax->rcount -= 2; /* * dl9sau bugfix: the trailling two bytes flexnet crc * will not be passed to the kernel. thus we have to * correct the kissparm signature, because it indicates * a crc but there's none */ *ax->rbuff &= ~0x20; } } count = ax->rcount; if ((skb = dev_alloc_skb(count)) == NULL) { printk(KERN_ERR "mkiss: %s: memory squeeze, dropping packet.\n", ax->dev->name); ax->dev->stats.rx_dropped++; spin_unlock_bh(&ax->buflock); return; } skb_put_data(skb, ax->rbuff, count); skb->protocol = ax25_type_trans(skb, ax->dev); netif_rx(skb); ax->dev->stats.rx_packets++; ax->dev->stats.rx_bytes += count; spin_unlock_bh(&ax->buflock); } static void kiss_unesc(struct mkiss *ax, unsigned char s) { switch (s) { case END: /* drop keeptest bit = VSV */ if (test_bit(AXF_KEEPTEST, &ax->flags)) clear_bit(AXF_KEEPTEST, &ax->flags); if (!test_and_clear_bit(AXF_ERROR, &ax->flags) && (ax->rcount > 2)) ax_bump(ax); clear_bit(AXF_ESCAPE, &ax->flags); ax->rcount = 0; return; case ESC: set_bit(AXF_ESCAPE, &ax->flags); return; case ESC_ESC: if (test_and_clear_bit(AXF_ESCAPE, &ax->flags)) s = ESC; break; case ESC_END: if (test_and_clear_bit(AXF_ESCAPE, &ax->flags)) s = END; break; } spin_lock_bh(&ax->buflock); if (!test_bit(AXF_ERROR, &ax->flags)) { if (ax->rcount < ax->buffsize) { ax->rbuff[ax->rcount++] = s; spin_unlock_bh(&ax->buflock); return; } ax->dev->stats.rx_over_errors++; set_bit(AXF_ERROR, &ax->flags); } spin_unlock_bh(&ax->buflock); } static int ax_set_mac_address(struct net_device *dev, void *addr) { struct sockaddr_ax25 *sa = addr; netif_tx_lock_bh(dev); netif_addr_lock(dev); __dev_addr_set(dev, &sa->sax25_call, AX25_ADDR_LEN); netif_addr_unlock(dev); netif_tx_unlock_bh(dev); return 0; } /*---------------------------------------------------------------------------*/ static void ax_changedmtu(struct mkiss *ax) { struct net_device *dev = ax->dev; unsigned char *xbuff, *rbuff, *oxbuff, *orbuff; int len; len = dev->mtu * 2; /* * allow for arrival of larger UDP packets, even if we say not to * also fixes a bug in which SunOS sends 512-byte packets even with * an MSS of 128 */ if (len < 576 * 2) len = 576 * 2; xbuff = kmalloc(len + 4, GFP_ATOMIC); rbuff = kmalloc(len + 4, GFP_ATOMIC); if (xbuff == NULL || rbuff == NULL) { printk(KERN_ERR "mkiss: %s: unable to grow ax25 buffers, " "MTU change cancelled.\n", ax->dev->name); dev->mtu = ax->mtu; kfree(xbuff); kfree(rbuff); return; } spin_lock_bh(&ax->buflock); oxbuff = ax->xbuff; ax->xbuff = xbuff; orbuff = ax->rbuff; ax->rbuff = rbuff; if (ax->xleft) { if (ax->xleft <= len) { memcpy(ax->xbuff, ax->xhead, ax->xleft); } else { ax->xleft = 0; dev->stats.tx_dropped++; } } ax->xhead = ax->xbuff; if (ax->rcount) { if (ax->rcount <= len) { memcpy(ax->rbuff, orbuff, ax->rcount); } else { ax->rcount = 0; dev->stats.rx_over_errors++; set_bit(AXF_ERROR, &ax->flags); } } ax->mtu = dev->mtu + 73; ax->buffsize = len; spin_unlock_bh(&ax->buflock); kfree(oxbuff); kfree(orbuff); } /* Encapsulate one AX.25 packet and stuff into a TTY queue. */ static void ax_encaps(struct net_device *dev, unsigned char *icp, int len) { struct mkiss *ax = netdev_priv(dev); unsigned char *p; int actual, count; if (ax->mtu != ax->dev->mtu + 73) /* Someone has been ifconfigging */ ax_changedmtu(ax); if (len > ax->mtu) { /* Sigh, shouldn't occur BUT ... */ printk(KERN_ERR "mkiss: %s: truncating oversized transmit packet!\n", ax->dev->name); dev->stats.tx_dropped++; netif_start_queue(dev); return; } p = icp; spin_lock_bh(&ax->buflock); if ((*p & 0x0f) != 0) { /* Configuration Command (kissparms(1). * Protocol spec says: never append CRC. * This fixes a very old bug in the linux * kiss driver. -- dl9sau */ switch (*p & 0xff) { case 0x85: /* command from userspace especially for us, * not for delivery to the tnc */ if (len > 1) { int cmd = (p[1] & 0xff); switch(cmd) { case 3: ax->crcmode = CRC_MODE_SMACK; break; case 2: ax->crcmode = CRC_MODE_FLEX; break; case 1: ax->crcmode = CRC_MODE_NONE; break; case 0: default: ax->crcmode = CRC_MODE_SMACK_TEST; cmd = 0; } ax->crcauto = (cmd ? 0 : 1); printk(KERN_INFO "mkiss: %s: crc mode set to %d\n", ax->dev->name, cmd); } spin_unlock_bh(&ax->buflock); netif_start_queue(dev); return; default: count = kiss_esc(p, ax->xbuff, len); } } else { unsigned short crc; switch (ax->crcmode) { case CRC_MODE_SMACK_TEST: ax->crcmode = CRC_MODE_FLEX_TEST; printk(KERN_INFO "mkiss: %s: Trying crc-smack\n", ax->dev->name); fallthrough; case CRC_MODE_SMACK: *p |= 0x80; crc = swab16(crc16(0, p, len)); count = kiss_esc_crc(p, ax->xbuff, crc, len+2); break; case CRC_MODE_FLEX_TEST: ax->crcmode = CRC_MODE_NONE; printk(KERN_INFO "mkiss: %s: Trying crc-flexnet\n", ax->dev->name); fallthrough; case CRC_MODE_FLEX: *p |= 0x20; crc = calc_crc_flex(p, len); count = kiss_esc_crc(p, ax->xbuff, crc, len+2); break; default: count = kiss_esc(p, ax->xbuff, len); } } spin_unlock_bh(&ax->buflock); set_bit(TTY_DO_WRITE_WAKEUP, &ax->tty->flags); actual = ax->tty->ops->write(ax->tty, ax->xbuff, count); dev->stats.tx_packets++; dev->stats.tx_bytes += actual; netif_trans_update(ax->dev); ax->xleft = count - actual; ax->xhead = ax->xbuff + actual; } /* Encapsulate an AX.25 packet and kick it into a TTY queue. */ static netdev_tx_t ax_xmit(struct sk_buff *skb, struct net_device *dev) { struct mkiss *ax = netdev_priv(dev); if (skb->protocol == htons(ETH_P_IP)) return ax25_ip_xmit(skb); if (!netif_running(dev)) { printk(KERN_ERR "mkiss: %s: xmit call when iface is down\n", dev->name); return NETDEV_TX_BUSY; } if (netif_queue_stopped(dev)) { /* * May be we must check transmitter timeout here ? * 14 Oct 1994 Dmitry Gorodchanin. */ if (time_before(jiffies, dev_trans_start(dev) + 20 * HZ)) { /* 20 sec timeout not reached */ return NETDEV_TX_BUSY; } printk(KERN_ERR "mkiss: %s: transmit timed out, %s?\n", dev->name, (tty_chars_in_buffer(ax->tty) || ax->xleft) ? "bad line quality" : "driver error"); ax->xleft = 0; clear_bit(TTY_DO_WRITE_WAKEUP, &ax->tty->flags); netif_start_queue(dev); } /* We were not busy, so we are now... :-) */ netif_stop_queue(dev); ax_encaps(dev, skb->data, skb->len); kfree_skb(skb); return NETDEV_TX_OK; } static int ax_open_dev(struct net_device *dev) { struct mkiss *ax = netdev_priv(dev); if (ax->tty == NULL) return -ENODEV; return 0; } /* Open the low-level part of the AX25 channel. Easy! */ static int ax_open(struct net_device *dev) { struct mkiss *ax = netdev_priv(dev); unsigned long len; if (ax->tty == NULL) return -ENODEV; /* * Allocate the frame buffers: * * rbuff Receive buffer. * xbuff Transmit buffer. */ len = dev->mtu * 2; /* * allow for arrival of larger UDP packets, even if we say not to * also fixes a bug in which SunOS sends 512-byte packets even with * an MSS of 128 */ if (len < 576 * 2) len = 576 * 2; if ((ax->rbuff = kmalloc(len + 4, GFP_KERNEL)) == NULL) goto norbuff; if ((ax->xbuff = kmalloc(len + 4, GFP_KERNEL)) == NULL) goto noxbuff; ax->mtu = dev->mtu + 73; ax->buffsize = len; ax->rcount = 0; ax->xleft = 0; ax->flags &= (1 << AXF_INUSE); /* Clear ESCAPE & ERROR flags */ spin_lock_init(&ax->buflock); return 0; noxbuff: kfree(ax->rbuff); norbuff: return -ENOMEM; } /* Close the low-level part of the AX25 channel. Easy! */ static int ax_close(struct net_device *dev) { struct mkiss *ax = netdev_priv(dev); if (ax->tty) clear_bit(TTY_DO_WRITE_WAKEUP, &ax->tty->flags); netif_stop_queue(dev); return 0; } static const struct net_device_ops ax_netdev_ops = { .ndo_open = ax_open_dev, .ndo_stop = ax_close, .ndo_start_xmit = ax_xmit, .ndo_set_mac_address = ax_set_mac_address, }; static void ax_setup(struct net_device *dev) { /* Finish setting up the DEVICE info. */ dev->mtu = AX_MTU; dev->hard_header_len = AX25_MAX_HEADER_LEN; dev->addr_len = AX25_ADDR_LEN; dev->type = ARPHRD_AX25; dev->tx_queue_len = 10; dev->header_ops = &ax25_header_ops; dev->netdev_ops = &ax_netdev_ops; memcpy(dev->broadcast, &ax25_bcast, AX25_ADDR_LEN); dev_addr_set(dev, (u8 *)&ax25_defaddr); dev->flags = IFF_BROADCAST | IFF_MULTICAST; } /* * We have a potential race on dereferencing tty->disc_data, because the tty * layer provides no locking at all - thus one cpu could be running * sixpack_receive_buf while another calls sixpack_close, which zeroes * tty->disc_data and frees the memory that sixpack_receive_buf is using. The * best way to fix this is to use a rwlock in the tty struct, but for now we * use a single global rwlock for all ttys in ppp line discipline. */ static DEFINE_RWLOCK(disc_data_lock); static struct mkiss *mkiss_get(struct tty_struct *tty) { struct mkiss *ax; read_lock(&disc_data_lock); ax = tty->disc_data; if (ax) refcount_inc(&ax->refcnt); read_unlock(&disc_data_lock); return ax; } static void mkiss_put(struct mkiss *ax) { if (refcount_dec_and_test(&ax->refcnt)) complete(&ax->dead); } static int crc_force = 0; /* Can be overridden with insmod */ static int mkiss_open(struct tty_struct *tty) { struct net_device *dev; struct mkiss *ax; int err; if (!capable(CAP_NET_ADMIN)) return -EPERM; if (tty->ops->write == NULL) return -EOPNOTSUPP; dev = alloc_netdev(sizeof(struct mkiss), "ax%d", NET_NAME_UNKNOWN, ax_setup); if (!dev) { err = -ENOMEM; goto out; } ax = netdev_priv(dev); ax->dev = dev; spin_lock_init(&ax->buflock); refcount_set(&ax->refcnt, 1); init_completion(&ax->dead); ax->tty = tty; tty->disc_data = ax; tty->receive_room = 65535; tty_driver_flush_buffer(tty); /* Restore default settings */ dev->type = ARPHRD_AX25; /* Perform the low-level AX25 initialization. */ err = ax_open(ax->dev); if (err) goto out_free_netdev; err = register_netdev(dev); if (err) goto out_free_buffers; /* after register_netdev() - because else printk smashes the kernel */ switch (crc_force) { case 3: ax->crcmode = CRC_MODE_SMACK; printk(KERN_INFO "mkiss: %s: crc mode smack forced.\n", ax->dev->name); break; case 2: ax->crcmode = CRC_MODE_FLEX; printk(KERN_INFO "mkiss: %s: crc mode flexnet forced.\n", ax->dev->name); break; case 1: ax->crcmode = CRC_MODE_NONE; printk(KERN_INFO "mkiss: %s: crc mode disabled.\n", ax->dev->name); break; case 0: default: crc_force = 0; printk(KERN_INFO "mkiss: %s: crc mode is auto.\n", ax->dev->name); ax->crcmode = CRC_MODE_SMACK_TEST; } ax->crcauto = (crc_force ? 0 : 1); netif_start_queue(dev); /* Done. We have linked the TTY line to a channel. */ return 0; out_free_buffers: kfree(ax->rbuff); kfree(ax->xbuff); out_free_netdev: free_netdev(dev); out: return err; } static void mkiss_close(struct tty_struct *tty) { struct mkiss *ax; write_lock_irq(&disc_data_lock); ax = tty->disc_data; tty->disc_data = NULL; write_unlock_irq(&disc_data_lock); if (!ax) return; /* * We have now ensured that nobody can start using ap from now on, but * we have to wait for all existing users to finish. */ if (!refcount_dec_and_test(&ax->refcnt)) wait_for_completion(&ax->dead); /* * Halt the transmit queue so that a new transmit cannot scribble * on our buffers */ netif_stop_queue(ax->dev); unregister_netdev(ax->dev); /* Free all AX25 frame buffers after unreg. */ kfree(ax->rbuff); kfree(ax->xbuff); ax->tty = NULL; free_netdev(ax->dev); } /* Perform I/O control on an active ax25 channel. */ static int mkiss_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { struct mkiss *ax = mkiss_get(tty); struct net_device *dev; unsigned int tmp, err; /* First make sure we're connected. */ if (ax == NULL) return -ENXIO; dev = ax->dev; switch (cmd) { case SIOCGIFNAME: err = copy_to_user((void __user *) arg, ax->dev->name, strlen(ax->dev->name) + 1) ? -EFAULT : 0; break; case SIOCGIFENCAP: err = put_user(4, (int __user *) arg); break; case SIOCSIFENCAP: if (get_user(tmp, (int __user *) arg)) { err = -EFAULT; break; } ax->mode = tmp; dev->addr_len = AX25_ADDR_LEN; dev->hard_header_len = AX25_KISS_HEADER_LEN + AX25_MAX_HEADER_LEN + 3; dev->type = ARPHRD_AX25; err = 0; break; case SIOCSIFHWADDR: { char addr[AX25_ADDR_LEN]; if (copy_from_user(&addr, (void __user *) arg, AX25_ADDR_LEN)) { err = -EFAULT; break; } netif_tx_lock_bh(dev); __dev_addr_set(dev, addr, AX25_ADDR_LEN); netif_tx_unlock_bh(dev); err = 0; break; } default: err = -ENOIOCTLCMD; } mkiss_put(ax); return err; } /* * Handle the 'receiver data ready' interrupt. * This function is called by the 'tty_io' module in the kernel when * a block of data has been received, which can now be decapsulated * and sent on to the AX.25 layer for further processing. */ static void mkiss_receive_buf(struct tty_struct *tty, const u8 *cp, const u8 *fp, size_t count) { struct mkiss *ax = mkiss_get(tty); if (!ax) return; /* * Argh! mtu change time! - costs us the packet part received * at the change */ if (ax->mtu != ax->dev->mtu + 73) ax_changedmtu(ax); /* Read the characters out of the buffer */ while (count--) { if (fp != NULL && *fp++) { if (!test_and_set_bit(AXF_ERROR, &ax->flags)) ax->dev->stats.rx_errors++; cp++; continue; } kiss_unesc(ax, *cp++); } mkiss_put(ax); tty_unthrottle(tty); } /* * Called by the driver when there's room for more data. If we have * more packets to send, we send them here. */ static void mkiss_write_wakeup(struct tty_struct *tty) { struct mkiss *ax = mkiss_get(tty); int actual; if (!ax) return; if (ax->xleft <= 0) { /* Now serial buffer is almost free & we can start * transmission of another packet */ clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); netif_wake_queue(ax->dev); goto out; } actual = tty->ops->write(tty, ax->xhead, ax->xleft); ax->xleft -= actual; ax->xhead += actual; out: mkiss_put(ax); } static struct tty_ldisc_ops ax_ldisc = { .owner = THIS_MODULE, .num = N_AX25, .name = "mkiss", .open = mkiss_open, .close = mkiss_close, .ioctl = mkiss_ioctl, .receive_buf = mkiss_receive_buf, .write_wakeup = mkiss_write_wakeup }; static const char banner[] __initconst = KERN_INFO \ "mkiss: AX.25 Multikiss, Hans Albas PE1AYX\n"; static const char msg_regfail[] __initconst = KERN_ERR \ "mkiss: can't register line discipline (err = %d)\n"; static int __init mkiss_init_driver(void) { int status; printk(banner); status = tty_register_ldisc(&ax_ldisc); if (status != 0) printk(msg_regfail, status); return status; } static void __exit mkiss_exit_driver(void) { tty_unregister_ldisc(&ax_ldisc); } MODULE_AUTHOR("Ralf Baechle DL5RB <ralf@linux-mips.org>"); MODULE_DESCRIPTION("KISS driver for AX.25 over TTYs"); module_param(crc_force, int, 0); MODULE_PARM_DESC(crc_force, "crc [0 = auto | 1 = none | 2 = flexnet | 3 = smack]"); MODULE_LICENSE("GPL"); MODULE_ALIAS_LDISC(N_AX25); module_init(mkiss_init_driver); module_exit(mkiss_exit_driver); |
| 27 23 28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 | #ifndef LLC_H #define LLC_H /* * Copyright (c) 1997 by Procom Technology, Inc. * 2001-2003 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> * * This program can be redistributed or modified under the terms of the * GNU General Public License as published by the Free Software Foundation. * This program is distributed without any warranty or implied warranty * of merchantability or fitness for a particular purpose. * * See the GNU General Public License for more details. */ #include <linux/if.h> #include <linux/if_ether.h> #include <linux/list.h> #include <linux/spinlock.h> #include <linux/rculist_nulls.h> #include <linux/hash.h> #include <linux/jhash.h> #include <linux/atomic.h> struct net_device; struct packet_type; struct sk_buff; struct llc_addr { unsigned char lsap; unsigned char mac[IFHWADDRLEN]; }; #define LLC_SAP_STATE_INACTIVE 1 #define LLC_SAP_STATE_ACTIVE 2 #define LLC_SK_DEV_HASH_BITS 6 #define LLC_SK_DEV_HASH_ENTRIES (1<<LLC_SK_DEV_HASH_BITS) #define LLC_SK_LADDR_HASH_BITS 6 #define LLC_SK_LADDR_HASH_ENTRIES (1<<LLC_SK_LADDR_HASH_BITS) /** * struct llc_sap - Defines the SAP component * * @station - station this sap belongs to * @state - sap state * @p_bit - only lowest-order bit used * @f_bit - only lowest-order bit used * @laddr - SAP value in this 'lsap' * @node - entry in station sap_list * @sk_list - LLC sockets this one manages */ struct llc_sap { unsigned char state; unsigned char p_bit; unsigned char f_bit; refcount_t refcnt; int (*rcv_func)(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev); struct llc_addr laddr; struct list_head node; spinlock_t sk_lock; int sk_count; struct hlist_nulls_head sk_laddr_hash[LLC_SK_LADDR_HASH_ENTRIES]; struct hlist_head sk_dev_hash[LLC_SK_DEV_HASH_ENTRIES]; struct rcu_head rcu; }; static inline struct hlist_head *llc_sk_dev_hash(struct llc_sap *sap, int ifindex) { u32 bucket = hash_32(ifindex, LLC_SK_DEV_HASH_BITS); return &sap->sk_dev_hash[bucket]; } static inline u32 llc_sk_laddr_hashfn(struct llc_sap *sap, const struct llc_addr *laddr) { return hash_32(jhash(laddr->mac, sizeof(laddr->mac), 0), LLC_SK_LADDR_HASH_BITS); } static inline struct hlist_nulls_head *llc_sk_laddr_hash(struct llc_sap *sap, const struct llc_addr *laddr) { return &sap->sk_laddr_hash[llc_sk_laddr_hashfn(sap, laddr)]; } #define LLC_DEST_INVALID 0 /* Invalid LLC PDU type */ #define LLC_DEST_SAP 1 /* Type 1 goes here */ #define LLC_DEST_CONN 2 /* Type 2 goes here */ extern struct list_head llc_sap_list; int llc_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev); int llc_mac_hdr_init(struct sk_buff *skb, const unsigned char *sa, const unsigned char *da); void llc_add_pack(int type, void (*handler)(struct llc_sap *sap, struct sk_buff *skb)); void llc_remove_pack(int type); void llc_set_station_handler(void (*handler)(struct sk_buff *skb)); struct llc_sap *llc_sap_open(unsigned char lsap, int (*rcv)(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev)); static inline void llc_sap_hold(struct llc_sap *sap) { refcount_inc(&sap->refcnt); } static inline bool llc_sap_hold_safe(struct llc_sap *sap) { return refcount_inc_not_zero(&sap->refcnt); } void llc_sap_close(struct llc_sap *sap); static inline void llc_sap_put(struct llc_sap *sap) { if (refcount_dec_and_test(&sap->refcnt)) llc_sap_close(sap); } struct llc_sap *llc_sap_find(unsigned char sap_value); int llc_build_and_send_ui_pkt(struct llc_sap *sap, struct sk_buff *skb, const unsigned char *dmac, unsigned char dsap); void llc_sap_handler(struct llc_sap *sap, struct sk_buff *skb); void llc_conn_handler(struct llc_sap *sap, struct sk_buff *skb); void llc_station_init(void); void llc_station_exit(void); #ifdef CONFIG_PROC_FS int llc_proc_init(void); void llc_proc_exit(void); #else #define llc_proc_init() (0) #define llc_proc_exit() do { } while(0) #endif /* CONFIG_PROC_FS */ #ifdef CONFIG_SYSCTL int llc_sysctl_init(void); void llc_sysctl_exit(void); extern int sysctl_llc2_ack_timeout; extern int sysctl_llc2_busy_timeout; extern int sysctl_llc2_p_timeout; extern int sysctl_llc2_rej_timeout; #else #define llc_sysctl_init() (0) #define llc_sysctl_exit() do { } while(0) #endif /* CONFIG_SYSCTL */ #endif /* LLC_H */ |
| 77 77 76 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 | // SPDX-License-Identifier: LGPL-2.0+ /* * Copyright (C) 1993, 1994, 1995, 1996, 1997 Free Software Foundation, Inc. * This file is part of the GNU C Library. * Contributed by Paul Eggert (eggert@twinsun.com). * * The GNU C Library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public License as * published by the Free Software Foundation; either version 2 of the * License, or (at your option) any later version. * * The GNU C Library is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Library General Public License for more details. * * You should have received a copy of the GNU Library General Public * License along with the GNU C Library; see the file COPYING.LIB. If not, * write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, * Boston, MA 02111-1307, USA. */ /* * Converts the calendar time to broken-down time representation * * 2009-7-14: * Moved from glibc-2.6 to kernel by Zhaolei<zhaolei@cn.fujitsu.com> * 2021-06-02: * Reimplemented by Cassio Neri <cassio.neri@gmail.com> */ #include <linux/time.h> #include <linux/module.h> #include <linux/kernel.h> #define SECS_PER_HOUR (60 * 60) #define SECS_PER_DAY (SECS_PER_HOUR * 24) /** * time64_to_tm - converts the calendar time to local broken-down time * * @totalsecs: the number of seconds elapsed since 00:00:00 on January 1, 1970, * Coordinated Universal Time (UTC). * @offset: offset seconds adding to totalsecs. * @result: pointer to struct tm variable to receive broken-down time */ void time64_to_tm(time64_t totalsecs, int offset, struct tm *result) { u32 u32tmp, day_of_century, year_of_century, day_of_year, month, day; u64 u64tmp, udays, century, year; bool is_Jan_or_Feb, is_leap_year; long days, rem; int remainder; days = div_s64_rem(totalsecs, SECS_PER_DAY, &remainder); rem = remainder; rem += offset; while (rem < 0) { rem += SECS_PER_DAY; --days; } while (rem >= SECS_PER_DAY) { rem -= SECS_PER_DAY; ++days; } result->tm_hour = rem / SECS_PER_HOUR; rem %= SECS_PER_HOUR; result->tm_min = rem / 60; result->tm_sec = rem % 60; /* January 1, 1970 was a Thursday. */ result->tm_wday = (4 + days) % 7; if (result->tm_wday < 0) result->tm_wday += 7; /* * The following algorithm is, basically, Proposition 6.3 of Neri * and Schneider [1]. In a few words: it works on the computational * (fictitious) calendar where the year starts in March, month = 2 * (*), and finishes in February, month = 13. This calendar is * mathematically convenient because the day of the year does not * depend on whether the year is leap or not. For instance: * * March 1st 0-th day of the year; * ... * April 1st 31-st day of the year; * ... * January 1st 306-th day of the year; (Important!) * ... * February 28th 364-th day of the year; * February 29th 365-th day of the year (if it exists). * * After having worked out the date in the computational calendar * (using just arithmetics) it's easy to convert it to the * corresponding date in the Gregorian calendar. * * [1] "Euclidean Affine Functions and Applications to Calendar * Algorithms". https://arxiv.org/abs/2102.06959 * * (*) The numbering of months follows tm more closely and thus, * is slightly different from [1]. */ udays = ((u64) days) + 2305843009213814918ULL; u64tmp = 4 * udays + 3; century = div64_u64_rem(u64tmp, 146097, &u64tmp); day_of_century = (u32) (u64tmp / 4); u32tmp = 4 * day_of_century + 3; u64tmp = 2939745ULL * u32tmp; year_of_century = upper_32_bits(u64tmp); day_of_year = lower_32_bits(u64tmp) / 2939745 / 4; year = 100 * century + year_of_century; is_leap_year = year_of_century ? !(year_of_century % 4) : !(century % 4); u32tmp = 2141 * day_of_year + 132377; month = u32tmp >> 16; day = ((u16) u32tmp) / 2141; /* * Recall that January 1st is the 306-th day of the year in the * computational (not Gregorian) calendar. */ is_Jan_or_Feb = day_of_year >= 306; /* Convert to the Gregorian calendar and adjust to Unix time. */ year = year + is_Jan_or_Feb - 6313183731940000ULL; month = is_Jan_or_Feb ? month - 12 : month; day = day + 1; day_of_year += is_Jan_or_Feb ? -306 : 31 + 28 + is_leap_year; /* Convert to tm's format. */ result->tm_year = (long) (year - 1900); result->tm_mon = (int) month; result->tm_mday = (int) day; result->tm_yday = (int) day_of_year; } EXPORT_SYMBOL(time64_to_tm); |
| 4 2 1 4 5 1 4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 | // SPDX-License-Identifier: GPL-2.0-or-later /* * STK1160 driver * * Copyright (C) 2012 Ezequiel Garcia * <elezegarcia--a.t--gmail.com> * * Based on Easycap driver by R.M. Thomas * Copyright (C) 2010 R.M. Thomas * <rmthomas--a.t--sciolus.org> * * TODO: * * 1. Support stream at lower speed: lower frame rate or lower frame size. */ #include <linux/module.h> #include <linux/init.h> #include <linux/kernel.h> #include <linux/errno.h> #include <linux/slab.h> #include <linux/usb.h> #include <linux/mm.h> #include <linux/vmalloc.h> #include <media/i2c/saa7115.h> #include "stk1160.h" #include "stk1160-reg.h" static unsigned int input; module_param(input, int, 0644); MODULE_PARM_DESC(input, "Set default input"); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Ezequiel Garcia"); MODULE_DESCRIPTION("STK1160 driver"); /* Devices supported by this driver */ static const struct usb_device_id stk1160_id_table[] = { { USB_DEVICE(0x05e1, 0x0408) }, { } }; MODULE_DEVICE_TABLE(usb, stk1160_id_table); /* saa7113 I2C address */ static unsigned short saa7113_addrs[] = { 0x4a >> 1, I2C_CLIENT_END }; /* * Read/Write stk registers */ int stk1160_read_reg(struct stk1160 *dev, u16 reg, u8 *value) { int ret; int pipe = usb_rcvctrlpipe(dev->udev, 0); u8 *buf; *value = 0; buf = kmalloc(sizeof(u8), GFP_KERNEL); if (!buf) return -ENOMEM; ret = usb_control_msg(dev->udev, pipe, 0x00, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, 0x00, reg, buf, sizeof(u8), 1000); if (ret < 0) { stk1160_err("read failed on reg 0x%x (%d)\n", reg, ret); kfree(buf); return ret; } *value = *buf; kfree(buf); return 0; } int stk1160_write_reg(struct stk1160 *dev, u16 reg, u16 value) { int ret; int pipe = usb_sndctrlpipe(dev->udev, 0); ret = usb_control_msg(dev->udev, pipe, 0x01, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE, value, reg, NULL, 0, 1000); if (ret < 0) { stk1160_err("write failed on reg 0x%x (%d)\n", reg, ret); return ret; } return 0; } void stk1160_select_input(struct stk1160 *dev) { int route; static const u8 gctrl[] = { 0x98, 0x90, 0x88, 0x80, 0x98 }; if (dev->ctl_input == STK1160_SVIDEO_INPUT) route = SAA7115_SVIDEO3; else route = SAA7115_COMPOSITE0; if (dev->ctl_input < ARRAY_SIZE(gctrl)) { v4l2_device_call_all(&dev->v4l2_dev, 0, video, s_routing, route, 0, 0); stk1160_write_reg(dev, STK1160_GCTRL, gctrl[dev->ctl_input]); } } /* TODO: We should break this into pieces */ static void stk1160_reg_reset(struct stk1160 *dev) { int i; static const struct regval ctl[] = { {STK1160_GCTRL+2, 0x0078}, {STK1160_RMCTL+1, 0x0000}, {STK1160_RMCTL+3, 0x0002}, {STK1160_PLLSO, 0x0010}, {STK1160_PLLSO+1, 0x0000}, {STK1160_PLLSO+2, 0x0014}, {STK1160_PLLSO+3, 0x000E}, {STK1160_PLLFD, 0x0046}, /* Timing generator setup */ {STK1160_TIGEN, 0x0012}, {STK1160_TICTL, 0x002D}, {STK1160_TICTL+1, 0x0001}, {STK1160_TICTL+2, 0x0000}, {STK1160_TICTL+3, 0x0000}, {STK1160_TIGEN, 0x0080}, {0xffff, 0xffff} }; for (i = 0; ctl[i].reg != 0xffff; i++) stk1160_write_reg(dev, ctl[i].reg, ctl[i].val); } static void stk1160_release(struct v4l2_device *v4l2_dev) { struct stk1160 *dev = container_of(v4l2_dev, struct stk1160, v4l2_dev); stk1160_dbg("releasing all resources\n"); stk1160_i2c_unregister(dev); v4l2_ctrl_handler_free(&dev->ctrl_handler); v4l2_device_unregister(&dev->v4l2_dev); mutex_destroy(&dev->v4l_lock); mutex_destroy(&dev->vb_queue_lock); kfree(dev->alt_max_pkt_size); kfree(dev); } /* high bandwidth multiplier, as encoded in highspeed endpoint descriptors */ #define hb_mult(wMaxPacketSize) (1 + (((wMaxPacketSize) >> 11) & 0x03)) /* * Scan usb interface and populate max_pkt_size array * with information on each alternate setting. * The array should be allocated by the caller. */ static int stk1160_scan_usb(struct usb_interface *intf, struct usb_device *udev, unsigned int *max_pkt_size) { int i, e, sizedescr, size, ifnum; const struct usb_endpoint_descriptor *desc; bool has_video = false, has_audio = false; const char *speed; ifnum = intf->altsetting[0].desc.bInterfaceNumber; /* Get endpoints */ for (i = 0; i < intf->num_altsetting; i++) { for (e = 0; e < intf->altsetting[i].desc.bNumEndpoints; e++) { /* This isn't clear enough, at least to me */ desc = &intf->altsetting[i].endpoint[e].desc; sizedescr = le16_to_cpu(desc->wMaxPacketSize); size = sizedescr & 0x7ff; if (udev->speed == USB_SPEED_HIGH) size = size * hb_mult(sizedescr); if (usb_endpoint_is_isoc_in(desc)) { switch (desc->bEndpointAddress) { case STK1160_EP_AUDIO: has_audio = true; break; case STK1160_EP_VIDEO: has_video = true; max_pkt_size[i] = size; break; } } } } /* Is this even possible? */ if (!(has_audio || has_video)) { dev_err(&udev->dev, "no audio or video endpoints found\n"); return -ENODEV; } switch (udev->speed) { case USB_SPEED_LOW: speed = "1.5"; break; case USB_SPEED_FULL: speed = "12"; break; case USB_SPEED_HIGH: speed = "480"; break; default: speed = "unknown"; } dev_info(&udev->dev, "New device %s %s @ %s Mbps (%04x:%04x, interface %d, class %d)\n", udev->manufacturer ? udev->manufacturer : "", udev->product ? udev->product : "", speed, le16_to_cpu(udev->descriptor.idVendor), le16_to_cpu(udev->descriptor.idProduct), ifnum, intf->altsetting->desc.bInterfaceNumber); /* This should never happen, since we rejected audio interfaces */ if (has_audio) dev_warn(&udev->dev, "audio interface %d found.\n\ This is not implemented by this driver,\ you should use snd-usb-audio instead\n", ifnum); if (has_video) dev_info(&udev->dev, "video interface %d found\n", ifnum); /* * Make sure we have 480 Mbps of bandwidth, otherwise things like * video stream wouldn't likely work, since 12 Mbps is generally * not enough even for most streams. */ if (udev->speed != USB_SPEED_HIGH) dev_warn(&udev->dev, "must be connected to a high-speed USB 2.0 port\n\ You may not be able to stream video smoothly\n"); return 0; } static int stk1160_probe(struct usb_interface *interface, const struct usb_device_id *id) { int rc = 0; unsigned int *alt_max_pkt_size; /* array of wMaxPacketSize */ struct usb_device *udev; struct stk1160 *dev; udev = interface_to_usbdev(interface); /* * Since usb audio class is supported by snd-usb-audio, * we reject audio interface. */ if (interface->altsetting[0].desc.bInterfaceClass == USB_CLASS_AUDIO) return -ENODEV; /* Alloc an array for all possible max_pkt_size */ alt_max_pkt_size = kmalloc_array(interface->num_altsetting, sizeof(alt_max_pkt_size[0]), GFP_KERNEL); if (alt_max_pkt_size == NULL) return -ENOMEM; /* * Scan usb possibilities and populate alt_max_pkt_size array. * Also, check if device speed is fast enough. */ rc = stk1160_scan_usb(interface, udev, alt_max_pkt_size); if (rc < 0) { kfree(alt_max_pkt_size); return rc; } dev = kzalloc(sizeof(struct stk1160), GFP_KERNEL); if (dev == NULL) { kfree(alt_max_pkt_size); return -ENOMEM; } dev->alt_max_pkt_size = alt_max_pkt_size; dev->udev = udev; dev->num_alt = interface->num_altsetting; dev->ctl_input = input; /* We save struct device for debug purposes only */ dev->dev = &interface->dev; usb_set_intfdata(interface, dev); /* initialize videobuf2 stuff */ rc = stk1160_vb2_setup(dev); if (rc < 0) goto free_err; /* * There is no need to take any locks here in probe * because we register the device node as the *last* thing. */ spin_lock_init(&dev->buf_lock); mutex_init(&dev->v4l_lock); mutex_init(&dev->vb_queue_lock); rc = v4l2_ctrl_handler_init(&dev->ctrl_handler, 0); if (rc) { stk1160_err("v4l2_ctrl_handler_init failed (%d)\n", rc); goto free_err; } /* * We obtain a v4l2_dev but defer * registration of video device node as the last thing. * There is no need to set the name if we give a device struct */ dev->v4l2_dev.release = stk1160_release; dev->v4l2_dev.ctrl_handler = &dev->ctrl_handler; rc = v4l2_device_register(dev->dev, &dev->v4l2_dev); if (rc) { stk1160_err("v4l2_device_register failed (%d)\n", rc); goto free_ctrl; } rc = stk1160_i2c_register(dev); if (rc < 0) goto unreg_v4l2; /* * To the best of my knowledge stk1160 boards only have * saa7113, but it doesn't hurt to support them all. */ dev->sd_saa7115 = v4l2_i2c_new_subdev(&dev->v4l2_dev, &dev->i2c_adap, "saa7115_auto", 0, saa7113_addrs); /* i2c reset saa711x */ v4l2_device_call_all(&dev->v4l2_dev, 0, core, reset, 0); v4l2_device_call_all(&dev->v4l2_dev, 0, video, s_stream, 0); /* reset stk1160 to default values */ stk1160_reg_reset(dev); /* select default input */ stk1160_select_input(dev); stk1160_ac97_setup(dev); rc = stk1160_video_register(dev); if (rc < 0) goto unreg_i2c; return 0; unreg_i2c: stk1160_i2c_unregister(dev); unreg_v4l2: v4l2_device_unregister(&dev->v4l2_dev); free_ctrl: v4l2_ctrl_handler_free(&dev->ctrl_handler); free_err: kfree(alt_max_pkt_size); kfree(dev); return rc; } static void stk1160_disconnect(struct usb_interface *interface) { struct stk1160 *dev; dev = usb_get_intfdata(interface); usb_set_intfdata(interface, NULL); /* * Wait until all current v4l2 operation are finished * then deallocate resources */ mutex_lock(&dev->vb_queue_lock); mutex_lock(&dev->v4l_lock); /* Here is the only place where isoc get released */ stk1160_uninit_isoc(dev); stk1160_clear_queue(dev, VB2_BUF_STATE_ERROR); video_unregister_device(&dev->vdev); v4l2_device_disconnect(&dev->v4l2_dev); /* This way current users can detect device is gone */ dev->udev = NULL; mutex_unlock(&dev->v4l_lock); mutex_unlock(&dev->vb_queue_lock); /* * This calls stk1160_release if it's the last reference. * Otherwise, release is postponed until there are no users left. */ v4l2_device_put(&dev->v4l2_dev); } static struct usb_driver stk1160_usb_driver = { .name = "stk1160", .id_table = stk1160_id_table, .probe = stk1160_probe, .disconnect = stk1160_disconnect, }; module_usb_driver(stk1160_usb_driver); |
| 43 43 5 5 84 83 43 83 83 13 1162 1154 1141 16 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 | // SPDX-License-Identifier: GPL-2.0-only /* Page fragment allocator * * Page Fragment: * An arbitrary-length arbitrary-offset area of memory which resides within a * 0 or higher order page. Multiple fragments within that page are * individually refcounted, in the page's reference counter. * * The page_frag functions provide a simple allocation framework for page * fragments. This is used by the network stack and network device drivers to * provide a backing region of memory for use as either an sk_buff->head, or to * be used in the "frags" portion of skb_shared_info. */ #include <linux/build_bug.h> #include <linux/export.h> #include <linux/gfp_types.h> #include <linux/init.h> #include <linux/mm.h> #include <linux/page_frag_cache.h> #include "internal.h" static unsigned long encoded_page_create(struct page *page, unsigned int order, bool pfmemalloc) { BUILD_BUG_ON(PAGE_FRAG_CACHE_MAX_ORDER > PAGE_FRAG_CACHE_ORDER_MASK); BUILD_BUG_ON(PAGE_FRAG_CACHE_PFMEMALLOC_BIT >= PAGE_SIZE); return (unsigned long)page_address(page) | (order & PAGE_FRAG_CACHE_ORDER_MASK) | ((unsigned long)pfmemalloc * PAGE_FRAG_CACHE_PFMEMALLOC_BIT); } static unsigned long encoded_page_decode_order(unsigned long encoded_page) { return encoded_page & PAGE_FRAG_CACHE_ORDER_MASK; } static void *encoded_page_decode_virt(unsigned long encoded_page) { return (void *)(encoded_page & PAGE_MASK); } static struct page *encoded_page_decode_page(unsigned long encoded_page) { return virt_to_page((void *)encoded_page); } static struct page *__page_frag_cache_refill(struct page_frag_cache *nc, gfp_t gfp_mask) { unsigned long order = PAGE_FRAG_CACHE_MAX_ORDER; struct page *page = NULL; gfp_t gfp = gfp_mask; #if (PAGE_SIZE < PAGE_FRAG_CACHE_MAX_SIZE) gfp_mask = (gfp_mask & ~__GFP_DIRECT_RECLAIM) | __GFP_COMP | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC; page = __alloc_pages(gfp_mask, PAGE_FRAG_CACHE_MAX_ORDER, numa_mem_id(), NULL); #endif if (unlikely(!page)) { page = __alloc_pages(gfp, 0, numa_mem_id(), NULL); order = 0; } nc->encoded_page = page ? encoded_page_create(page, order, page_is_pfmemalloc(page)) : 0; return page; } void page_frag_cache_drain(struct page_frag_cache *nc) { if (!nc->encoded_page) return; __page_frag_cache_drain(encoded_page_decode_page(nc->encoded_page), nc->pagecnt_bias); nc->encoded_page = 0; } EXPORT_SYMBOL(page_frag_cache_drain); void __page_frag_cache_drain(struct page *page, unsigned int count) { VM_BUG_ON_PAGE(page_ref_count(page) == 0, page); if (page_ref_sub_and_test(page, count)) free_frozen_pages(page, compound_order(page)); } EXPORT_SYMBOL(__page_frag_cache_drain); void *__page_frag_alloc_align(struct page_frag_cache *nc, unsigned int fragsz, gfp_t gfp_mask, unsigned int align_mask) { unsigned long encoded_page = nc->encoded_page; unsigned int size, offset; struct page *page; if (unlikely(!encoded_page)) { refill: page = __page_frag_cache_refill(nc, gfp_mask); if (!page) return NULL; encoded_page = nc->encoded_page; /* Even if we own the page, we do not use atomic_set(). * This would break get_page_unless_zero() users. */ page_ref_add(page, PAGE_FRAG_CACHE_MAX_SIZE); /* reset page count bias and offset to start of new frag */ nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; nc->offset = 0; } size = PAGE_SIZE << encoded_page_decode_order(encoded_page); offset = __ALIGN_KERNEL_MASK(nc->offset, ~align_mask); if (unlikely(offset + fragsz > size)) { if (unlikely(fragsz > PAGE_SIZE)) { /* * The caller is trying to allocate a fragment * with fragsz > PAGE_SIZE but the cache isn't big * enough to satisfy the request, this may * happen in low memory conditions. * We don't release the cache page because * it could make memory pressure worse * so we simply return NULL here. */ return NULL; } page = encoded_page_decode_page(encoded_page); if (!page_ref_sub_and_test(page, nc->pagecnt_bias)) goto refill; if (unlikely(encoded_page_decode_pfmemalloc(encoded_page))) { free_frozen_pages(page, encoded_page_decode_order(encoded_page)); goto refill; } /* OK, page count is 0, we can safely set it */ set_page_count(page, PAGE_FRAG_CACHE_MAX_SIZE + 1); /* reset page count bias and offset to start of new frag */ nc->pagecnt_bias = PAGE_FRAG_CACHE_MAX_SIZE + 1; offset = 0; } nc->pagecnt_bias--; nc->offset = offset + fragsz; return encoded_page_decode_virt(encoded_page) + offset; } EXPORT_SYMBOL(__page_frag_alloc_align); /* * Frees a page fragment allocated out of either a compound or order 0 page. */ void page_frag_free(void *addr) { struct page *page = virt_to_head_page(addr); if (unlikely(put_page_testzero(page))) free_frozen_pages(page, compound_order(page)); } EXPORT_SYMBOL(page_frag_free); |
| 12 11 1 6 3 5 5 3 5 5 7 7 6 6 6 6 4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 | // SPDX-License-Identifier: GPL-2.0-only #include "netlink.h" #include "common.h" struct pause_req_info { struct ethnl_req_info base; enum ethtool_mac_stats_src src; }; #define PAUSE_REQINFO(__req_base) \ container_of(__req_base, struct pause_req_info, base) struct pause_reply_data { struct ethnl_reply_data base; struct ethtool_pauseparam pauseparam; struct ethtool_pause_stats pausestat; }; #define PAUSE_REPDATA(__reply_base) \ container_of(__reply_base, struct pause_reply_data, base) const struct nla_policy ethnl_pause_get_policy[] = { [ETHTOOL_A_PAUSE_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy_stats), [ETHTOOL_A_PAUSE_STATS_SRC] = NLA_POLICY_MAX(NLA_U32, ETHTOOL_MAC_STATS_SRC_PMAC), }; static int pause_parse_request(struct ethnl_req_info *req_base, struct nlattr **tb, struct netlink_ext_ack *extack) { enum ethtool_mac_stats_src src = ETHTOOL_MAC_STATS_SRC_AGGREGATE; struct pause_req_info *req_info = PAUSE_REQINFO(req_base); if (tb[ETHTOOL_A_PAUSE_STATS_SRC]) { if (!(req_base->flags & ETHTOOL_FLAG_STATS)) { NL_SET_ERR_MSG_MOD(extack, "ETHTOOL_FLAG_STATS must be set when requesting a source of stats"); return -EINVAL; } src = nla_get_u32(tb[ETHTOOL_A_PAUSE_STATS_SRC]); } req_info->src = src; return 0; } static int pause_prepare_data(const struct ethnl_req_info *req_base, struct ethnl_reply_data *reply_base, const struct genl_info *info) { const struct pause_req_info *req_info = PAUSE_REQINFO(req_base); struct pause_reply_data *data = PAUSE_REPDATA(reply_base); enum ethtool_mac_stats_src src = req_info->src; struct net_device *dev = reply_base->dev; int ret; if (!dev->ethtool_ops->get_pauseparam) return -EOPNOTSUPP; ethtool_stats_init((u64 *)&data->pausestat, sizeof(data->pausestat) / 8); data->pausestat.src = src; ret = ethnl_ops_begin(dev); if (ret < 0) return ret; if ((src == ETHTOOL_MAC_STATS_SRC_EMAC || src == ETHTOOL_MAC_STATS_SRC_PMAC) && !__ethtool_dev_mm_supported(dev)) { NL_SET_ERR_MSG_MOD(info->extack, "Device does not support MAC merge layer"); ethnl_ops_complete(dev); return -EOPNOTSUPP; } dev->ethtool_ops->get_pauseparam(dev, &data->pauseparam); if (req_base->flags & ETHTOOL_FLAG_STATS && dev->ethtool_ops->get_pause_stats) dev->ethtool_ops->get_pause_stats(dev, &data->pausestat); ethnl_ops_complete(dev); return 0; } static int pause_reply_size(const struct ethnl_req_info *req_base, const struct ethnl_reply_data *reply_base) { int n = nla_total_size(sizeof(u8)) + /* _PAUSE_AUTONEG */ nla_total_size(sizeof(u8)) + /* _PAUSE_RX */ nla_total_size(sizeof(u8)); /* _PAUSE_TX */ if (req_base->flags & ETHTOOL_FLAG_STATS) n += nla_total_size(0) + /* _PAUSE_STATS */ nla_total_size(sizeof(u32)) + /* _PAUSE_STATS_SRC */ nla_total_size_64bit(sizeof(u64)) * ETHTOOL_PAUSE_STAT_CNT; return n; } static int ethtool_put_stat(struct sk_buff *skb, u64 val, u16 attrtype, u16 padtype) { if (val == ETHTOOL_STAT_NOT_SET) return 0; if (nla_put_u64_64bit(skb, attrtype, val, padtype)) return -EMSGSIZE; return 0; } static int pause_put_stats(struct sk_buff *skb, const struct ethtool_pause_stats *pause_stats) { const u16 pad = ETHTOOL_A_PAUSE_STAT_PAD; struct nlattr *nest; if (nla_put_u32(skb, ETHTOOL_A_PAUSE_STATS_SRC, pause_stats->src)) return -EMSGSIZE; nest = nla_nest_start(skb, ETHTOOL_A_PAUSE_STATS); if (!nest) return -EMSGSIZE; if (ethtool_put_stat(skb, pause_stats->tx_pause_frames, ETHTOOL_A_PAUSE_STAT_TX_FRAMES, pad) || ethtool_put_stat(skb, pause_stats->rx_pause_frames, ETHTOOL_A_PAUSE_STAT_RX_FRAMES, pad)) goto err_cancel; nla_nest_end(skb, nest); return 0; err_cancel: nla_nest_cancel(skb, nest); return -EMSGSIZE; } static int pause_fill_reply(struct sk_buff *skb, const struct ethnl_req_info *req_base, const struct ethnl_reply_data *reply_base) { const struct pause_reply_data *data = PAUSE_REPDATA(reply_base); const struct ethtool_pauseparam *pauseparam = &data->pauseparam; if (nla_put_u8(skb, ETHTOOL_A_PAUSE_AUTONEG, !!pauseparam->autoneg) || nla_put_u8(skb, ETHTOOL_A_PAUSE_RX, !!pauseparam->rx_pause) || nla_put_u8(skb, ETHTOOL_A_PAUSE_TX, !!pauseparam->tx_pause)) return -EMSGSIZE; if (req_base->flags & ETHTOOL_FLAG_STATS && pause_put_stats(skb, &data->pausestat)) return -EMSGSIZE; return 0; } /* PAUSE_SET */ const struct nla_policy ethnl_pause_set_policy[] = { [ETHTOOL_A_PAUSE_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), [ETHTOOL_A_PAUSE_AUTONEG] = { .type = NLA_U8 }, [ETHTOOL_A_PAUSE_RX] = { .type = NLA_U8 }, [ETHTOOL_A_PAUSE_TX] = { .type = NLA_U8 }, [ETHTOOL_A_PAUSE_STATS_SRC] = { .type = NLA_REJECT }, }; static int ethnl_set_pause_validate(struct ethnl_req_info *req_info, struct genl_info *info) { const struct ethtool_ops *ops = req_info->dev->ethtool_ops; return ops->get_pauseparam && ops->set_pauseparam ? 1 : -EOPNOTSUPP; } static int ethnl_set_pause(struct ethnl_req_info *req_info, struct genl_info *info) { struct net_device *dev = req_info->dev; struct ethtool_pauseparam params = {}; struct nlattr **tb = info->attrs; bool mod = false; int ret; dev->ethtool_ops->get_pauseparam(dev, ¶ms); ethnl_update_bool32(¶ms.autoneg, tb[ETHTOOL_A_PAUSE_AUTONEG], &mod); ethnl_update_bool32(¶ms.rx_pause, tb[ETHTOOL_A_PAUSE_RX], &mod); ethnl_update_bool32(¶ms.tx_pause, tb[ETHTOOL_A_PAUSE_TX], &mod); if (!mod) return 0; ret = dev->ethtool_ops->set_pauseparam(dev, ¶ms); return ret < 0 ? ret : 1; } const struct ethnl_request_ops ethnl_pause_request_ops = { .request_cmd = ETHTOOL_MSG_PAUSE_GET, .reply_cmd = ETHTOOL_MSG_PAUSE_GET_REPLY, .hdr_attr = ETHTOOL_A_PAUSE_HEADER, .req_info_size = sizeof(struct pause_req_info), .reply_data_size = sizeof(struct pause_reply_data), .parse_request = pause_parse_request, .prepare_data = pause_prepare_data, .reply_size = pause_reply_size, .fill_reply = pause_fill_reply, .set_validate = ethnl_set_pause_validate, .set = ethnl_set_pause, .set_ntf_cmd = ETHTOOL_MSG_PAUSE_NTF, }; |
| 1528 8 1517 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 | // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (c) 2007-2012 Nicira, Inc. */ #include <linux/netdevice.h> #include <net/genetlink.h> #include <net/netns/generic.h> #include "datapath.h" #include "vport-internal_dev.h" #include "vport-netdev.h" static void dp_detach_port_notify(struct vport *vport) { struct sk_buff *notify; struct datapath *dp; dp = vport->dp; notify = ovs_vport_cmd_build_info(vport, ovs_dp_get_net(dp), 0, 0, OVS_VPORT_CMD_DEL); ovs_dp_detach_port(vport); if (IS_ERR(notify)) { genl_set_err(&dp_vport_genl_family, ovs_dp_get_net(dp), 0, 0, PTR_ERR(notify)); return; } genlmsg_multicast_netns(&dp_vport_genl_family, ovs_dp_get_net(dp), notify, 0, 0, GFP_KERNEL); } void ovs_dp_notify_wq(struct work_struct *work) { struct ovs_net *ovs_net = container_of(work, struct ovs_net, dp_notify_work); struct datapath *dp; ovs_lock(); list_for_each_entry(dp, &ovs_net->dps, list_node) { int i; for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) { struct vport *vport; struct hlist_node *n; hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node) { if (vport->ops->type == OVS_VPORT_TYPE_INTERNAL) continue; if (!(netif_is_ovs_port(vport->dev))) dp_detach_port_notify(vport); } } } ovs_unlock(); } static int dp_device_event(struct notifier_block *unused, unsigned long event, void *ptr) { struct ovs_net *ovs_net; struct net_device *dev = netdev_notifier_info_to_dev(ptr); struct vport *vport = NULL; if (!ovs_is_internal_dev(dev)) vport = ovs_netdev_get_vport(dev); if (!vport) return NOTIFY_DONE; if (event == NETDEV_UNREGISTER) { /* upper_dev_unlink and decrement promisc immediately */ ovs_netdev_detach_dev(vport); /* schedule vport destroy, dev_put and genl notification */ ovs_net = net_generic(dev_net(dev), ovs_net_id); queue_work(system_percpu_wq, &ovs_net->dp_notify_work); } return NOTIFY_DONE; } struct notifier_block ovs_dp_device_notifier = { .notifier_call = dp_device_event }; |
| 1 1 1 5 5 5 4 4 6 1 1 6 6 4 4 5 5 5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 | // SPDX-License-Identifier: GPL-2.0 #include <linux/kernel.h> #include <linux/module.h> #include <linux/backing-dev.h> #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/mm.h> #include <linux/init.h> #include <linux/slab.h> #include <linux/workqueue.h> #include <linux/smp.h> #include "blk.h" #include "blk-mq.h" static void blk_mq_sysfs_release(struct kobject *kobj) { struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj); free_percpu(ctxs->queue_ctx); kfree(ctxs); } static void blk_mq_ctx_sysfs_release(struct kobject *kobj) { struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj); /* ctx->ctxs won't be released until all ctx are freed */ kobject_put(&ctx->ctxs->kobj); } static void blk_mq_hw_sysfs_release(struct kobject *kobj) { struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); sbitmap_free(&hctx->ctx_map); free_cpumask_var(hctx->cpumask); kfree(hctx->ctxs); kfree(hctx); } struct blk_mq_hw_ctx_sysfs_entry { struct attribute attr; ssize_t (*show)(struct blk_mq_hw_ctx *, char *); }; static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj, struct attribute *attr, char *page) { struct blk_mq_hw_ctx_sysfs_entry *entry; struct blk_mq_hw_ctx *hctx; struct request_queue *q; ssize_t res; entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr); hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj); q = hctx->queue; if (!entry->show) return -EIO; mutex_lock(&q->elevator_lock); res = entry->show(hctx, page); mutex_unlock(&q->elevator_lock); return res; } static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx, char *page) { return sprintf(page, "%u\n", hctx->tags->nr_tags); } static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx, char *page) { return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags); } static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page) { const size_t size = PAGE_SIZE - 1; unsigned int i, first = 1; int ret = 0, pos = 0; for_each_cpu(i, hctx->cpumask) { if (first) ret = snprintf(pos + page, size - pos, "%u", i); else ret = snprintf(pos + page, size - pos, ", %u", i); if (ret >= size - pos) break; first = 0; pos += ret; } ret = snprintf(pos + page, size + 1 - pos, "\n"); return pos + ret; } static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = { .attr = {.name = "nr_tags", .mode = 0444 }, .show = blk_mq_hw_sysfs_nr_tags_show, }; static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = { .attr = {.name = "nr_reserved_tags", .mode = 0444 }, .show = blk_mq_hw_sysfs_nr_reserved_tags_show, }; static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = { .attr = {.name = "cpu_list", .mode = 0444 }, .show = blk_mq_hw_sysfs_cpus_show, }; static struct attribute *default_hw_ctx_attrs[] = { &blk_mq_hw_sysfs_nr_tags.attr, &blk_mq_hw_sysfs_nr_reserved_tags.attr, &blk_mq_hw_sysfs_cpus.attr, NULL, }; ATTRIBUTE_GROUPS(default_hw_ctx); static const struct sysfs_ops blk_mq_hw_sysfs_ops = { .show = blk_mq_hw_sysfs_show, }; static const struct kobj_type blk_mq_ktype = { .release = blk_mq_sysfs_release, }; static const struct kobj_type blk_mq_ctx_ktype = { .release = blk_mq_ctx_sysfs_release, }; static const struct kobj_type blk_mq_hw_ktype = { .sysfs_ops = &blk_mq_hw_sysfs_ops, .default_groups = default_hw_ctx_groups, .release = blk_mq_hw_sysfs_release, }; static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx) { struct blk_mq_ctx *ctx; int i; if (!hctx->nr_ctx) return; hctx_for_each_ctx(hctx, ctx, i) if (ctx->kobj.state_in_sysfs) kobject_del(&ctx->kobj); if (hctx->kobj.state_in_sysfs) kobject_del(&hctx->kobj); } static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx) { struct request_queue *q = hctx->queue; struct blk_mq_ctx *ctx; int i, j, ret; if (!hctx->nr_ctx) return 0; ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num); if (ret) return ret; hctx_for_each_ctx(hctx, ctx, i) { ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu); if (ret) goto out; } return 0; out: hctx_for_each_ctx(hctx, ctx, j) { if (j < i) kobject_del(&ctx->kobj); } kobject_del(&hctx->kobj); return ret; } void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx) { kobject_init(&hctx->kobj, &blk_mq_hw_ktype); } void blk_mq_sysfs_deinit(struct request_queue *q) { struct blk_mq_ctx *ctx; int cpu; for_each_possible_cpu(cpu) { ctx = per_cpu_ptr(q->queue_ctx, cpu); kobject_put(&ctx->kobj); } kobject_put(q->mq_kobj); } void blk_mq_sysfs_init(struct request_queue *q) { struct blk_mq_ctx *ctx; int cpu; kobject_init(q->mq_kobj, &blk_mq_ktype); for_each_possible_cpu(cpu) { ctx = per_cpu_ptr(q->queue_ctx, cpu); kobject_get(q->mq_kobj); kobject_init(&ctx->kobj, &blk_mq_ctx_ktype); } } int blk_mq_sysfs_register(struct gendisk *disk) { struct request_queue *q = disk->queue; struct blk_mq_hw_ctx *hctx; unsigned long i, j; int ret; ret = kobject_add(q->mq_kobj, &disk_to_dev(disk)->kobj, "mq"); if (ret < 0) return ret; kobject_uevent(q->mq_kobj, KOBJ_ADD); mutex_lock(&q->tag_set->tag_list_lock); queue_for_each_hw_ctx(q, hctx, i) { ret = blk_mq_register_hctx(hctx); if (ret) goto out_unreg; } mutex_unlock(&q->tag_set->tag_list_lock); return 0; out_unreg: queue_for_each_hw_ctx(q, hctx, j) { if (j < i) blk_mq_unregister_hctx(hctx); } mutex_unlock(&q->tag_set->tag_list_lock); kobject_uevent(q->mq_kobj, KOBJ_REMOVE); kobject_del(q->mq_kobj); return ret; } void blk_mq_sysfs_unregister(struct gendisk *disk) { struct request_queue *q = disk->queue; struct blk_mq_hw_ctx *hctx; unsigned long i; mutex_lock(&q->tag_set->tag_list_lock); queue_for_each_hw_ctx(q, hctx, i) blk_mq_unregister_hctx(hctx); mutex_unlock(&q->tag_set->tag_list_lock); kobject_uevent(q->mq_kobj, KOBJ_REMOVE); kobject_del(q->mq_kobj); } void blk_mq_sysfs_unregister_hctxs(struct request_queue *q) { struct blk_mq_hw_ctx *hctx; unsigned long i; if (!blk_queue_registered(q)) return; queue_for_each_hw_ctx(q, hctx, i) blk_mq_unregister_hctx(hctx); } int blk_mq_sysfs_register_hctxs(struct request_queue *q) { struct blk_mq_hw_ctx *hctx; unsigned long i; int ret = 0; if (!blk_queue_registered(q)) goto out; queue_for_each_hw_ctx(q, hctx, i) { ret = blk_mq_register_hctx(hctx); if (ret) break; } out: return ret; } |
| 104 2 3 28 23 7 6 65 99 23 12 74 6 36 38 40 18 17 10 674 676 675 671 670 8 8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | // SPDX-License-Identifier: GPL-2.0-or-later /* * common UDP/RAW code * Linux INET implementation * * Authors: * Hideaki YOSHIFUJI <yoshfuji@linux-ipv6.org> */ #include <linux/types.h> #include <linux/module.h> #include <linux/in.h> #include <net/ip.h> #include <net/sock.h> #include <net/route.h> #include <net/tcp_states.h> #include <net/sock_reuseport.h> int __ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) { struct inet_sock *inet = inet_sk(sk); struct sockaddr_in *usin = (struct sockaddr_in *) uaddr; struct flowi4 *fl4; struct rtable *rt; __be32 saddr; int oif; int err; if (addr_len < sizeof(*usin)) return -EINVAL; if (usin->sin_family != AF_INET) return -EAFNOSUPPORT; sk_dst_reset(sk); oif = sk->sk_bound_dev_if; saddr = inet->inet_saddr; if (ipv4_is_multicast(usin->sin_addr.s_addr)) { if (!oif || netif_index_is_l3_master(sock_net(sk), oif)) oif = READ_ONCE(inet->mc_index); if (!saddr) saddr = READ_ONCE(inet->mc_addr); } else if (!oif) { oif = READ_ONCE(inet->uc_index); } fl4 = &inet->cork.fl.u.ip4; rt = ip_route_connect(fl4, usin->sin_addr.s_addr, saddr, oif, sk->sk_protocol, inet->inet_sport, usin->sin_port, sk); if (IS_ERR(rt)) { err = PTR_ERR(rt); if (err == -ENETUNREACH) IP_INC_STATS(sock_net(sk), IPSTATS_MIB_OUTNOROUTES); goto out; } if ((rt->rt_flags & RTCF_BROADCAST) && !sock_flag(sk, SOCK_BROADCAST)) { ip_rt_put(rt); err = -EACCES; goto out; } /* Update addresses before rehashing */ inet->inet_daddr = fl4->daddr; inet->inet_dport = usin->sin_port; if (!inet->inet_saddr) inet->inet_saddr = fl4->saddr; if (!inet->inet_rcv_saddr) { inet->inet_rcv_saddr = fl4->saddr; if (sk->sk_prot->rehash) sk->sk_prot->rehash(sk); } reuseport_has_conns_set(sk); sk->sk_state = TCP_ESTABLISHED; sk_set_txhash(sk); atomic_set(&inet->inet_id, get_random_u16()); sk_dst_set(sk, &rt->dst); err = 0; out: return err; } EXPORT_SYMBOL(__ip4_datagram_connect); int ip4_datagram_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) { int res; lock_sock(sk); res = __ip4_datagram_connect(sk, uaddr, addr_len); release_sock(sk); return res; } EXPORT_SYMBOL(ip4_datagram_connect); /* Because UDP xmit path can manipulate sk_dst_cache without holding * socket lock, we need to use sk_dst_set() here, * even if we own the socket lock. */ void ip4_datagram_release_cb(struct sock *sk) { const struct inet_sock *inet = inet_sk(sk); struct dst_entry *dst; struct flowi4 fl4; struct rtable *rt; rcu_read_lock(); dst = __sk_dst_get(sk); if (!dst || !READ_ONCE(dst->obsolete) || dst->ops->check(dst, 0)) { rcu_read_unlock(); return; } inet_sk_init_flowi4(inet, &fl4); rt = ip_route_output_flow(sock_net(sk), &fl4, sk); dst = !IS_ERR(rt) ? &rt->dst : NULL; sk_dst_set(sk, dst); rcu_read_unlock(); } EXPORT_SYMBOL_GPL(ip4_datagram_release_cb); |
| 8 1 16 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 | /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2007, 2008, 2009 Siemens AG * * Written by: * Dmitry Eremin-Solenikov <dbaryshkov@gmail.com> */ #ifndef __NET_CFG802154_H #define __NET_CFG802154_H #include <linux/ieee802154.h> #include <linux/netdevice.h> #include <linux/spinlock.h> #include <linux/bug.h> #include <net/nl802154.h> struct wpan_phy; struct wpan_phy_cca; struct cfg802154_scan_request; struct cfg802154_beacon_request; struct ieee802154_addr; #ifdef CONFIG_IEEE802154_NL802154_EXPERIMENTAL struct ieee802154_llsec_device_key; struct ieee802154_llsec_seclevel; struct ieee802154_llsec_params; struct ieee802154_llsec_device; struct ieee802154_llsec_table; struct ieee802154_llsec_key_id; struct ieee802154_llsec_key; #endif /* CONFIG_IEEE802154_NL802154_EXPERIMENTAL */ struct cfg802154_ops { struct net_device * (*add_virtual_intf_deprecated)(struct wpan_phy *wpan_phy, const char *name, unsigned char name_assign_type, int type); void (*del_virtual_intf_deprecated)(struct wpan_phy *wpan_phy, struct net_device *dev); int (*suspend)(struct wpan_phy *wpan_phy); int (*resume)(struct wpan_phy *wpan_phy); int (*add_virtual_intf)(struct wpan_phy *wpan_phy, const char *name, unsigned char name_assign_type, enum nl802154_iftype type, __le64 extended_addr); int (*del_virtual_intf)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev); int (*set_channel)(struct wpan_phy *wpan_phy, u8 page, u8 channel); int (*set_cca_mode)(struct wpan_phy *wpan_phy, const struct wpan_phy_cca *cca); int (*set_cca_ed_level)(struct wpan_phy *wpan_phy, s32 ed_level); int (*set_tx_power)(struct wpan_phy *wpan_phy, s32 power); int (*set_pan_id)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, __le16 pan_id); int (*set_short_addr)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, __le16 short_addr); int (*set_backoff_exponent)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, u8 min_be, u8 max_be); int (*set_max_csma_backoffs)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, u8 max_csma_backoffs); int (*set_max_frame_retries)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, s8 max_frame_retries); int (*set_lbt_mode)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, bool mode); int (*set_ackreq_default)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, bool ackreq); int (*trigger_scan)(struct wpan_phy *wpan_phy, struct cfg802154_scan_request *request); int (*abort_scan)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev); int (*send_beacons)(struct wpan_phy *wpan_phy, struct cfg802154_beacon_request *request); int (*stop_beacons)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev); int (*associate)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, struct ieee802154_addr *coord); int (*disassociate)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, struct ieee802154_addr *target); #ifdef CONFIG_IEEE802154_NL802154_EXPERIMENTAL void (*get_llsec_table)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, struct ieee802154_llsec_table **table); void (*lock_llsec_table)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev); void (*unlock_llsec_table)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev); /* TODO remove locking/get table callbacks, this is part of the * nl802154 interface and should be accessible from ieee802154 layer. */ int (*get_llsec_params)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, struct ieee802154_llsec_params *params); int (*set_llsec_params)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, const struct ieee802154_llsec_params *params, int changed); int (*add_llsec_key)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, const struct ieee802154_llsec_key_id *id, const struct ieee802154_llsec_key *key); int (*del_llsec_key)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, const struct ieee802154_llsec_key_id *id); int (*add_seclevel)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, const struct ieee802154_llsec_seclevel *sl); int (*del_seclevel)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, const struct ieee802154_llsec_seclevel *sl); int (*add_device)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, const struct ieee802154_llsec_device *dev); int (*del_device)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, __le64 extended_addr); int (*add_devkey)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, __le64 extended_addr, const struct ieee802154_llsec_device_key *key); int (*del_devkey)(struct wpan_phy *wpan_phy, struct wpan_dev *wpan_dev, __le64 extended_addr, const struct ieee802154_llsec_device_key *key); #endif /* CONFIG_IEEE802154_NL802154_EXPERIMENTAL */ }; static inline bool wpan_phy_supported_bool(bool b, enum nl802154_supported_bool_states st) { switch (st) { case NL802154_SUPPORTED_BOOL_TRUE: return b; case NL802154_SUPPORTED_BOOL_FALSE: return !b; case NL802154_SUPPORTED_BOOL_BOTH: return true; default: WARN_ON(1); } return false; } struct wpan_phy_supported { u32 channels[IEEE802154_MAX_PAGE + 1], cca_modes, cca_opts, iftypes; enum nl802154_supported_bool_states lbt; u8 min_minbe, max_minbe, min_maxbe, max_maxbe, min_csma_backoffs, max_csma_backoffs; s8 min_frame_retries, max_frame_retries; size_t tx_powers_size, cca_ed_levels_size; const s32 *tx_powers, *cca_ed_levels; }; struct wpan_phy_cca { enum nl802154_cca_modes mode; enum nl802154_cca_opts opt; }; static inline bool wpan_phy_cca_cmp(const struct wpan_phy_cca *a, const struct wpan_phy_cca *b) { if (a->mode != b->mode) return false; if (a->mode == NL802154_CCA_ENERGY_CARRIER) return a->opt == b->opt; return true; } /** * enum wpan_phy_flags - WPAN PHY state flags * @WPAN_PHY_FLAG_TXPOWER: Indicates that transceiver will support * transmit power setting. * @WPAN_PHY_FLAG_CCA_ED_LEVEL: Indicates that transceiver will support cca ed * level setting. * @WPAN_PHY_FLAG_CCA_MODE: Indicates that transceiver will support cca mode * setting. * @WPAN_PHY_FLAG_STATE_QUEUE_STOPPED: Indicates that the transmit queue was * temporarily stopped. * @WPAN_PHY_FLAG_DATAGRAMS_ONLY: Indicates that transceiver is only able to * send/receive datagrams. */ enum wpan_phy_flags { WPAN_PHY_FLAG_TXPOWER = BIT(1), WPAN_PHY_FLAG_CCA_ED_LEVEL = BIT(2), WPAN_PHY_FLAG_CCA_MODE = BIT(3), WPAN_PHY_FLAG_STATE_QUEUE_STOPPED = BIT(4), WPAN_PHY_FLAG_DATAGRAMS_ONLY = BIT(5), }; struct wpan_phy { /* If multiple wpan_phys are registered and you're handed e.g. * a regular netdev with assigned ieee802154_ptr, you won't * know whether it points to a wpan_phy your driver has registered * or not. Assign this to something global to your driver to * help determine whether you own this wpan_phy or not. */ const void *privid; unsigned long flags; /* * This is a PIB according to 802.15.4-2011. * We do not provide timing-related variables, as they * aren't used outside of driver */ u8 current_channel; u8 current_page; struct wpan_phy_supported supported; /* current transmit_power in mBm */ s32 transmit_power; struct wpan_phy_cca cca; __le64 perm_extended_addr; /* current cca ed threshold in mBm */ s32 cca_ed_level; /* PHY depended MAC PIB values */ /* 802.15.4 acronym: Tdsym in nsec */ u32 symbol_duration; /* lifs and sifs periods timing */ u16 lifs_period; u16 sifs_period; struct device dev; /* the network namespace this phy lives in currently */ possible_net_t _net; /* Transmission monitoring and control */ spinlock_t queue_lock; atomic_t ongoing_txs; atomic_t hold_txs; wait_queue_head_t sync_txq; /* Current filtering level on reception. * Only allowed to be changed if phy is not operational. */ enum ieee802154_filtering_level filtering; char priv[] __aligned(NETDEV_ALIGN); }; static inline struct net *wpan_phy_net(struct wpan_phy *wpan_phy) { return read_pnet(&wpan_phy->_net); } static inline void wpan_phy_net_set(struct wpan_phy *wpan_phy, struct net *net) { write_pnet(&wpan_phy->_net, net); } static inline bool ieee802154_chan_is_valid(struct wpan_phy *phy, u8 page, u8 channel) { if (page > IEEE802154_MAX_PAGE || channel > IEEE802154_MAX_CHANNEL || !(phy->supported.channels[page] & BIT(channel))) return false; return true; } /** * struct ieee802154_addr - IEEE802.15.4 device address * @mode: Address mode from frame header. Can be one of: * - @IEEE802154_ADDR_NONE * - @IEEE802154_ADDR_SHORT * - @IEEE802154_ADDR_LONG * @pan_id: The PAN ID this address belongs to * @short_addr: address if @mode is @IEEE802154_ADDR_SHORT * @extended_addr: address if @mode is @IEEE802154_ADDR_LONG */ struct ieee802154_addr { u8 mode; __le16 pan_id; union { __le16 short_addr; __le64 extended_addr; }; }; /** * struct ieee802154_coord_desc - Coordinator descriptor * @addr: PAN ID and coordinator address * @page: page this coordinator is using * @channel: channel this coordinator is using * @superframe_spec: SuperFrame specification as received * @link_quality: link quality indicator at which the beacon was received * @gts_permit: the coordinator accepts GTS requests */ struct ieee802154_coord_desc { struct ieee802154_addr addr; u8 page; u8 channel; u16 superframe_spec; u8 link_quality; bool gts_permit; }; /** * struct ieee802154_pan_device - PAN device information * @pan_id: the PAN ID of this device * @mode: the preferred mode to reach the device * @short_addr: the short address of this device * @extended_addr: the extended address of this device * @node: the list node */ struct ieee802154_pan_device { __le16 pan_id; u8 mode; __le16 short_addr; __le64 extended_addr; struct list_head node; }; /** * struct cfg802154_scan_request - Scan request * * @type: type of scan to be performed * @page: page on which to perform the scan * @channels: channels in te %page to be scanned * @duration: time spent on each channel, calculated with: * aBaseSuperframeDuration * (2 ^ duration + 1) * @wpan_dev: the wpan device on which to perform the scan * @wpan_phy: the wpan phy on which to perform the scan */ struct cfg802154_scan_request { enum nl802154_scan_types type; u8 page; u32 channels; u8 duration; struct wpan_dev *wpan_dev; struct wpan_phy *wpan_phy; }; /** * struct cfg802154_beacon_request - Beacon request descriptor * * @interval: interval n between sendings, in multiple order of the super frame * duration: aBaseSuperframeDuration * (2^n) unless the interval * order is greater or equal to 15, in this case beacons won't be * passively sent out at a fixed rate but instead inform the device * that it should answer beacon requests as part of active scan * procedures * @wpan_dev: the concerned wpan device * @wpan_phy: the wpan phy this was for */ struct cfg802154_beacon_request { u8 interval; struct wpan_dev *wpan_dev; struct wpan_phy *wpan_phy; }; /** * struct cfg802154_mac_pkt - MAC packet descriptor (beacon/command) * @node: MAC packets to process list member * @skb: the received sk_buff * @sdata: the interface on which @skb was received * @page: page configuration when @skb was received * @channel: channel configuration when @skb was received */ struct cfg802154_mac_pkt { struct list_head node; struct sk_buff *skb; struct ieee802154_sub_if_data *sdata; u8 page; u8 channel; }; struct ieee802154_llsec_key_id { u8 mode; u8 id; union { struct ieee802154_addr device_addr; __le32 short_source; __le64 extended_source; }; }; #define IEEE802154_LLSEC_KEY_SIZE 16 struct ieee802154_llsec_key { u8 frame_types; u32 cmd_frame_ids; /* TODO replace with NL802154_KEY_SIZE */ u8 key[IEEE802154_LLSEC_KEY_SIZE]; }; struct ieee802154_llsec_key_entry { struct list_head list; struct rcu_head rcu; struct ieee802154_llsec_key_id id; struct ieee802154_llsec_key *key; }; struct ieee802154_llsec_params { bool enabled; __be32 frame_counter; u8 out_level; struct ieee802154_llsec_key_id out_key; __le64 default_key_source; __le16 pan_id; __le64 hwaddr; __le64 coord_hwaddr; __le16 coord_shortaddr; }; struct ieee802154_llsec_table { struct list_head keys; struct list_head devices; struct list_head security_levels; }; struct ieee802154_llsec_seclevel { struct list_head list; u8 frame_type; u8 cmd_frame_id; bool device_override; u32 sec_levels; }; struct ieee802154_llsec_device { struct list_head list; __le16 pan_id; __le16 short_addr; __le64 hwaddr; u32 frame_counter; bool seclevel_exempt; u8 key_mode; struct list_head keys; }; struct ieee802154_llsec_device_key { struct list_head list; struct ieee802154_llsec_key_id key_id; u32 frame_counter; }; struct wpan_dev_header_ops { /* TODO create callback currently assumes ieee802154_mac_cb inside * skb->cb. This should be changed to give these information as * parameter. */ int (*create)(struct sk_buff *skb, struct net_device *dev, const struct ieee802154_addr *daddr, const struct ieee802154_addr *saddr, unsigned int len); }; struct wpan_dev { struct wpan_phy *wpan_phy; int iftype; /* the remainder of this struct should be private to cfg802154 */ struct list_head list; struct net_device *netdev; const struct wpan_dev_header_ops *header_ops; /* lowpan interface, set when the wpan_dev belongs to one lowpan_dev */ struct net_device *lowpan_dev; u32 identifier; /* MAC PIB */ __le16 pan_id; __le16 short_addr; __le64 extended_addr; /* MAC BSN field */ atomic_t bsn; /* MAC DSN field */ atomic_t dsn; u8 min_be; u8 max_be; u8 csma_retries; s8 frame_retries; bool lbt; /* fallback for acknowledgment bit setting */ bool ackreq; /* Associations */ struct mutex association_lock; struct ieee802154_pan_device *parent; struct list_head children; unsigned int max_associations; unsigned int nchildren; }; #define to_phy(_dev) container_of(_dev, struct wpan_phy, dev) #if IS_ENABLED(CONFIG_IEEE802154) || IS_ENABLED(CONFIG_6LOWPAN) static inline int wpan_dev_hard_header(struct sk_buff *skb, struct net_device *dev, const struct ieee802154_addr *daddr, const struct ieee802154_addr *saddr, unsigned int len) { struct wpan_dev *wpan_dev = dev->ieee802154_ptr; return wpan_dev->header_ops->create(skb, dev, daddr, saddr, len); } #endif struct wpan_phy * wpan_phy_new(const struct cfg802154_ops *ops, size_t priv_size); static inline void wpan_phy_set_dev(struct wpan_phy *phy, struct device *dev) { phy->dev.parent = dev; } int wpan_phy_register(struct wpan_phy *phy); void wpan_phy_unregister(struct wpan_phy *phy); void wpan_phy_free(struct wpan_phy *phy); /* Same semantics as for class_for_each_device */ int wpan_phy_for_each(int (*fn)(struct wpan_phy *phy, void *data), void *data); static inline void *wpan_phy_priv(struct wpan_phy *phy) { BUG_ON(!phy); return &phy->priv; } struct wpan_phy *wpan_phy_find(const char *str); static inline void wpan_phy_put(struct wpan_phy *phy) { put_device(&phy->dev); } static inline const char *wpan_phy_name(struct wpan_phy *phy) { return dev_name(&phy->dev); } void ieee802154_configure_durations(struct wpan_phy *phy, unsigned int page, unsigned int channel); /** * cfg802154_device_is_associated - Checks whether we are associated to any device * @wpan_dev: the wpan device * @return: true if we are associated */ bool cfg802154_device_is_associated(struct wpan_dev *wpan_dev); /** * cfg802154_device_is_parent - Checks if a device is our coordinator * @wpan_dev: the wpan device * @target: the expected parent * @return: true if @target is our coordinator */ bool cfg802154_device_is_parent(struct wpan_dev *wpan_dev, struct ieee802154_addr *target); /** * cfg802154_device_is_child - Checks whether a device is associated to us * @wpan_dev: the wpan device * @target: the expected child * @return: the PAN device */ struct ieee802154_pan_device * cfg802154_device_is_child(struct wpan_dev *wpan_dev, struct ieee802154_addr *target); /** * cfg802154_set_max_associations - Limit the number of future associations * @wpan_dev: the wpan device * @max: the maximum number of devices we accept to associate * @return: the old maximum value */ unsigned int cfg802154_set_max_associations(struct wpan_dev *wpan_dev, unsigned int max); /** * cfg802154_get_free_short_addr - Get a free address among the known devices * @wpan_dev: the wpan device * @return: a random short address expectedly unused on our PAN */ __le16 cfg802154_get_free_short_addr(struct wpan_dev *wpan_dev); #endif /* __NET_CFG802154_H */ |
| 3 3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Force feedback support for Logitech Gaming Wheels * * Including G27, G25, DFP, DFGT, FFEX, Momo, Momo2 & * Speed Force Wireless (WiiWheel) * * Copyright (c) 2010 Simon Wood <simon@mungewell.org> */ /* */ #include <linux/input.h> #include <linux/usb.h> #include <linux/hid.h> #include "usbhid/usbhid.h" #include "hid-lg.h" #include "hid-lg4ff.h" #include "hid-ids.h" #define LG4FF_MMODE_IS_MULTIMODE 0 #define LG4FF_MMODE_SWITCHED 1 #define LG4FF_MMODE_NOT_MULTIMODE 2 #define LG4FF_MODE_NATIVE_IDX 0 #define LG4FF_MODE_DFEX_IDX 1 #define LG4FF_MODE_DFP_IDX 2 #define LG4FF_MODE_G25_IDX 3 #define LG4FF_MODE_DFGT_IDX 4 #define LG4FF_MODE_G27_IDX 5 #define LG4FF_MODE_G29_IDX 6 #define LG4FF_MODE_MAX_IDX 7 #define LG4FF_MODE_NATIVE BIT(LG4FF_MODE_NATIVE_IDX) #define LG4FF_MODE_DFEX BIT(LG4FF_MODE_DFEX_IDX) #define LG4FF_MODE_DFP BIT(LG4FF_MODE_DFP_IDX) #define LG4FF_MODE_G25 BIT(LG4FF_MODE_G25_IDX) #define LG4FF_MODE_DFGT BIT(LG4FF_MODE_DFGT_IDX) #define LG4FF_MODE_G27 BIT(LG4FF_MODE_G27_IDX) #define LG4FF_MODE_G29 BIT(LG4FF_MODE_G29_IDX) #define LG4FF_DFEX_TAG "DF-EX" #define LG4FF_DFEX_NAME "Driving Force / Formula EX" #define LG4FF_DFP_TAG "DFP" #define LG4FF_DFP_NAME "Driving Force Pro" #define LG4FF_G25_TAG "G25" #define LG4FF_G25_NAME "G25 Racing Wheel" #define LG4FF_G27_TAG "G27" #define LG4FF_G27_NAME "G27 Racing Wheel" #define LG4FF_G29_TAG "G29" #define LG4FF_G29_NAME "G29 Racing Wheel" #define LG4FF_DFGT_TAG "DFGT" #define LG4FF_DFGT_NAME "Driving Force GT" #define LG4FF_FFEX_REV_MAJ 0x21 #define LG4FF_FFEX_REV_MIN 0x00 static void lg4ff_set_range_dfp(struct hid_device *hid, u16 range); static void lg4ff_set_range_g25(struct hid_device *hid, u16 range); struct lg4ff_wheel_data { const u32 product_id; u16 combine; u16 range; const u16 min_range; const u16 max_range; #ifdef CONFIG_LEDS_CLASS u8 led_state; struct led_classdev *led[5]; #endif const u32 alternate_modes; const char * const real_tag; const char * const real_name; const u16 real_product_id; void (*set_range)(struct hid_device *hid, u16 range); }; struct lg4ff_device_entry { spinlock_t report_lock; /* Protect output HID report */ struct hid_report *report; struct lg4ff_wheel_data wdata; }; static const signed short lg4ff_wheel_effects[] = { FF_CONSTANT, FF_AUTOCENTER, -1 }; static const signed short no_wheel_effects[] = { -1 }; struct lg4ff_wheel { const u32 product_id; const signed short *ff_effects; const u16 min_range; const u16 max_range; void (*set_range)(struct hid_device *hid, u16 range); }; struct lg4ff_compat_mode_switch { const u8 cmd_count; /* Number of commands to send */ const u8 cmd[]; }; struct lg4ff_wheel_ident_info { const u32 modes; const u16 mask; const u16 result; const u16 real_product_id; }; struct lg4ff_multimode_wheel { const u16 product_id; const u32 alternate_modes; const char *real_tag; const char *real_name; }; struct lg4ff_alternate_mode { const u16 product_id; const char *tag; const char *name; }; static const struct lg4ff_wheel lg4ff_devices[] = { {USB_DEVICE_ID_LOGITECH_WINGMAN_FG, no_wheel_effects, 40, 180, NULL}, {USB_DEVICE_ID_LOGITECH_WINGMAN_FFG, lg4ff_wheel_effects, 40, 180, NULL}, {USB_DEVICE_ID_LOGITECH_WHEEL, lg4ff_wheel_effects, 40, 270, NULL}, {USB_DEVICE_ID_LOGITECH_MOMO_WHEEL, lg4ff_wheel_effects, 40, 270, NULL}, {USB_DEVICE_ID_LOGITECH_DFP_WHEEL, lg4ff_wheel_effects, 40, 900, lg4ff_set_range_dfp}, {USB_DEVICE_ID_LOGITECH_G25_WHEEL, lg4ff_wheel_effects, 40, 900, lg4ff_set_range_g25}, {USB_DEVICE_ID_LOGITECH_DFGT_WHEEL, lg4ff_wheel_effects, 40, 900, lg4ff_set_range_g25}, {USB_DEVICE_ID_LOGITECH_G27_WHEEL, lg4ff_wheel_effects, 40, 900, lg4ff_set_range_g25}, {USB_DEVICE_ID_LOGITECH_G29_WHEEL, lg4ff_wheel_effects, 40, 900, lg4ff_set_range_g25}, {USB_DEVICE_ID_LOGITECH_MOMO_WHEEL2, lg4ff_wheel_effects, 40, 270, NULL}, {USB_DEVICE_ID_LOGITECH_WII_WHEEL, lg4ff_wheel_effects, 40, 270, NULL} }; static const struct lg4ff_multimode_wheel lg4ff_multimode_wheels[] = { {USB_DEVICE_ID_LOGITECH_DFP_WHEEL, LG4FF_MODE_NATIVE | LG4FF_MODE_DFP | LG4FF_MODE_DFEX, LG4FF_DFP_TAG, LG4FF_DFP_NAME}, {USB_DEVICE_ID_LOGITECH_G25_WHEEL, LG4FF_MODE_NATIVE | LG4FF_MODE_G25 | LG4FF_MODE_DFP | LG4FF_MODE_DFEX, LG4FF_G25_TAG, LG4FF_G25_NAME}, {USB_DEVICE_ID_LOGITECH_DFGT_WHEEL, LG4FF_MODE_NATIVE | LG4FF_MODE_DFGT | LG4FF_MODE_DFP | LG4FF_MODE_DFEX, LG4FF_DFGT_TAG, LG4FF_DFGT_NAME}, {USB_DEVICE_ID_LOGITECH_G27_WHEEL, LG4FF_MODE_NATIVE | LG4FF_MODE_G27 | LG4FF_MODE_G25 | LG4FF_MODE_DFP | LG4FF_MODE_DFEX, LG4FF_G27_TAG, LG4FF_G27_NAME}, {USB_DEVICE_ID_LOGITECH_G29_WHEEL, LG4FF_MODE_NATIVE | LG4FF_MODE_G29 | LG4FF_MODE_G27 | LG4FF_MODE_G25 | LG4FF_MODE_DFGT | LG4FF_MODE_DFP | LG4FF_MODE_DFEX, LG4FF_G29_TAG, LG4FF_G29_NAME}, }; static const struct lg4ff_alternate_mode lg4ff_alternate_modes[] = { [LG4FF_MODE_NATIVE_IDX] = {0, "native", ""}, [LG4FF_MODE_DFEX_IDX] = {USB_DEVICE_ID_LOGITECH_WHEEL, LG4FF_DFEX_TAG, LG4FF_DFEX_NAME}, [LG4FF_MODE_DFP_IDX] = {USB_DEVICE_ID_LOGITECH_DFP_WHEEL, LG4FF_DFP_TAG, LG4FF_DFP_NAME}, [LG4FF_MODE_G25_IDX] = {USB_DEVICE_ID_LOGITECH_G25_WHEEL, LG4FF_G25_TAG, LG4FF_G25_NAME}, [LG4FF_MODE_DFGT_IDX] = {USB_DEVICE_ID_LOGITECH_DFGT_WHEEL, LG4FF_DFGT_TAG, LG4FF_DFGT_NAME}, [LG4FF_MODE_G27_IDX] = {USB_DEVICE_ID_LOGITECH_G27_WHEEL, LG4FF_G27_TAG, LG4FF_G27_NAME}, [LG4FF_MODE_G29_IDX] = {USB_DEVICE_ID_LOGITECH_G29_WHEEL, LG4FF_G29_TAG, LG4FF_G29_NAME}, }; /* Multimode wheel identificators */ static const struct lg4ff_wheel_ident_info lg4ff_dfp_ident_info = { LG4FF_MODE_DFP | LG4FF_MODE_DFEX, 0xf000, 0x1000, USB_DEVICE_ID_LOGITECH_DFP_WHEEL }; static const struct lg4ff_wheel_ident_info lg4ff_g25_ident_info = { LG4FF_MODE_G25 | LG4FF_MODE_DFP | LG4FF_MODE_DFEX, 0xff00, 0x1200, USB_DEVICE_ID_LOGITECH_G25_WHEEL }; static const struct lg4ff_wheel_ident_info lg4ff_g27_ident_info = { LG4FF_MODE_G27 | LG4FF_MODE_G25 | LG4FF_MODE_DFP | LG4FF_MODE_DFEX, 0xfff0, 0x1230, USB_DEVICE_ID_LOGITECH_G27_WHEEL }; static const struct lg4ff_wheel_ident_info lg4ff_dfgt_ident_info = { LG4FF_MODE_DFGT | LG4FF_MODE_DFP | LG4FF_MODE_DFEX, 0xff00, 0x1300, USB_DEVICE_ID_LOGITECH_DFGT_WHEEL }; static const struct lg4ff_wheel_ident_info lg4ff_g29_ident_info = { LG4FF_MODE_G29 | LG4FF_MODE_G27 | LG4FF_MODE_G25 | LG4FF_MODE_DFGT | LG4FF_MODE_DFP | LG4FF_MODE_DFEX, 0xfff8, 0x1350, USB_DEVICE_ID_LOGITECH_G29_WHEEL }; static const struct lg4ff_wheel_ident_info lg4ff_g29_ident_info2 = { LG4FF_MODE_G29 | LG4FF_MODE_G27 | LG4FF_MODE_G25 | LG4FF_MODE_DFGT | LG4FF_MODE_DFP | LG4FF_MODE_DFEX, 0xff00, 0x8900, USB_DEVICE_ID_LOGITECH_G29_WHEEL }; /* Multimode wheel identification checklists */ static const struct lg4ff_wheel_ident_info *lg4ff_main_checklist[] = { &lg4ff_g29_ident_info, &lg4ff_g29_ident_info2, &lg4ff_dfgt_ident_info, &lg4ff_g27_ident_info, &lg4ff_g25_ident_info, &lg4ff_dfp_ident_info }; /* Compatibility mode switching commands */ /* EXT_CMD9 - Understood by G27 and DFGT */ static const struct lg4ff_compat_mode_switch lg4ff_mode_switch_ext09_dfex = { 2, {0xf8, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, /* Revert mode upon USB reset */ 0xf8, 0x09, 0x00, 0x01, 0x00, 0x00, 0x00} /* Switch mode to DF-EX with detach */ }; static const struct lg4ff_compat_mode_switch lg4ff_mode_switch_ext09_dfp = { 2, {0xf8, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, /* Revert mode upon USB reset */ 0xf8, 0x09, 0x01, 0x01, 0x00, 0x00, 0x00} /* Switch mode to DFP with detach */ }; static const struct lg4ff_compat_mode_switch lg4ff_mode_switch_ext09_g25 = { 2, {0xf8, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, /* Revert mode upon USB reset */ 0xf8, 0x09, 0x02, 0x01, 0x00, 0x00, 0x00} /* Switch mode to G25 with detach */ }; static const struct lg4ff_compat_mode_switch lg4ff_mode_switch_ext09_dfgt = { 2, {0xf8, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, /* Revert mode upon USB reset */ 0xf8, 0x09, 0x03, 0x01, 0x00, 0x00, 0x00} /* Switch mode to DFGT with detach */ }; static const struct lg4ff_compat_mode_switch lg4ff_mode_switch_ext09_g27 = { 2, {0xf8, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, /* Revert mode upon USB reset */ 0xf8, 0x09, 0x04, 0x01, 0x00, 0x00, 0x00} /* Switch mode to G27 with detach */ }; static const struct lg4ff_compat_mode_switch lg4ff_mode_switch_ext09_g29 = { 2, {0xf8, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, /* Revert mode upon USB reset */ 0xf8, 0x09, 0x05, 0x01, 0x01, 0x00, 0x00} /* Switch mode to G29 with detach */ }; /* EXT_CMD1 - Understood by DFP, G25, G27 and DFGT */ static const struct lg4ff_compat_mode_switch lg4ff_mode_switch_ext01_dfp = { 1, {0xf8, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00} }; /* EXT_CMD16 - Understood by G25 and G27 */ static const struct lg4ff_compat_mode_switch lg4ff_mode_switch_ext16_g25 = { 1, {0xf8, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00} }; /* Recalculates X axis value accordingly to currently selected range */ static s32 lg4ff_adjust_dfp_x_axis(s32 value, u16 range) { u16 max_range; s32 new_value; if (range == 900) return value; else if (range == 200) return value; else if (range < 200) max_range = 200; else max_range = 900; new_value = 8192 + mult_frac(value - 8192, max_range, range); if (new_value < 0) return 0; else if (new_value > 16383) return 16383; else return new_value; } int lg4ff_adjust_input_event(struct hid_device *hid, struct hid_field *field, struct hid_usage *usage, s32 value, struct lg_drv_data *drv_data) { struct lg4ff_device_entry *entry = drv_data->device_props; s32 new_value = 0; if (!entry) { hid_err(hid, "Device properties not found"); return 0; } switch (entry->wdata.product_id) { case USB_DEVICE_ID_LOGITECH_DFP_WHEEL: switch (usage->code) { case ABS_X: new_value = lg4ff_adjust_dfp_x_axis(value, entry->wdata.range); input_event(field->hidinput->input, usage->type, usage->code, new_value); return 1; default: return 0; } default: return 0; } } int lg4ff_raw_event(struct hid_device *hdev, struct hid_report *report, u8 *rd, int size, struct lg_drv_data *drv_data) { int offset; struct lg4ff_device_entry *entry = drv_data->device_props; if (!entry) return 0; /* adjust HID report present combined pedals data */ if (entry->wdata.combine) { switch (entry->wdata.product_id) { case USB_DEVICE_ID_LOGITECH_WHEEL: rd[5] = rd[3]; rd[6] = 0x7F; return 1; case USB_DEVICE_ID_LOGITECH_WINGMAN_FG: case USB_DEVICE_ID_LOGITECH_WINGMAN_FFG: case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL: case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL2: rd[4] = rd[3]; rd[5] = 0x7F; return 1; case USB_DEVICE_ID_LOGITECH_DFP_WHEEL: rd[5] = rd[4]; rd[6] = 0x7F; return 1; case USB_DEVICE_ID_LOGITECH_G25_WHEEL: case USB_DEVICE_ID_LOGITECH_G27_WHEEL: offset = 5; break; case USB_DEVICE_ID_LOGITECH_DFGT_WHEEL: case USB_DEVICE_ID_LOGITECH_G29_WHEEL: offset = 6; break; case USB_DEVICE_ID_LOGITECH_WII_WHEEL: offset = 3; break; default: return 0; } /* Compute a combined axis when wheel does not supply it */ rd[offset] = (0xFF + rd[offset] - rd[offset+1]) >> 1; rd[offset+1] = 0x7F; return 1; } return 0; } static void lg4ff_init_wheel_data(struct lg4ff_wheel_data * const wdata, const struct lg4ff_wheel *wheel, const struct lg4ff_multimode_wheel *mmode_wheel, const u16 real_product_id) { u32 alternate_modes = 0; const char *real_tag = NULL; const char *real_name = NULL; if (mmode_wheel) { alternate_modes = mmode_wheel->alternate_modes; real_tag = mmode_wheel->real_tag; real_name = mmode_wheel->real_name; } { struct lg4ff_wheel_data t_wdata = { .product_id = wheel->product_id, .real_product_id = real_product_id, .combine = 0, .min_range = wheel->min_range, .max_range = wheel->max_range, .set_range = wheel->set_range, .alternate_modes = alternate_modes, .real_tag = real_tag, .real_name = real_name }; memcpy(wdata, &t_wdata, sizeof(t_wdata)); } } static int lg4ff_play(struct input_dev *dev, void *data, struct ff_effect *effect) { struct hid_device *hid = input_get_drvdata(dev); struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; unsigned long flags; s32 *value; int x; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return -EINVAL; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return -EINVAL; } value = entry->report->field[0]->value; #define CLAMP(x) do { if (x < 0) x = 0; else if (x > 0xff) x = 0xff; } while (0) switch (effect->type) { case FF_CONSTANT: x = effect->u.ramp.start_level + 0x80; /* 0x80 is no force */ CLAMP(x); spin_lock_irqsave(&entry->report_lock, flags); if (x == 0x80) { /* De-activate force in slot-1*/ value[0] = 0x13; value[1] = 0x00; value[2] = 0x00; value[3] = 0x00; value[4] = 0x00; value[5] = 0x00; value[6] = 0x00; hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); spin_unlock_irqrestore(&entry->report_lock, flags); return 0; } value[0] = 0x11; /* Slot 1 */ value[1] = 0x08; value[2] = x; value[3] = 0x80; value[4] = 0x00; value[5] = 0x00; value[6] = 0x00; hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); spin_unlock_irqrestore(&entry->report_lock, flags); break; } return 0; } /* Sends default autocentering command compatible with * all wheels except Formula Force EX */ static void lg4ff_set_autocenter_default(struct input_dev *dev, u16 magnitude) { struct hid_device *hid = input_get_drvdata(dev); s32 *value; u32 expand_a, expand_b; struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; unsigned long flags; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return; } value = entry->report->field[0]->value; /* De-activate Auto-Center */ spin_lock_irqsave(&entry->report_lock, flags); if (magnitude == 0) { value[0] = 0xf5; value[1] = 0x00; value[2] = 0x00; value[3] = 0x00; value[4] = 0x00; value[5] = 0x00; value[6] = 0x00; hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); spin_unlock_irqrestore(&entry->report_lock, flags); return; } if (magnitude <= 0xaaaa) { expand_a = 0x0c * magnitude; expand_b = 0x80 * magnitude; } else { expand_a = (0x0c * 0xaaaa) + 0x06 * (magnitude - 0xaaaa); expand_b = (0x80 * 0xaaaa) + 0xff * (magnitude - 0xaaaa); } /* Adjust for non-MOMO wheels */ switch (entry->wdata.product_id) { case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL: case USB_DEVICE_ID_LOGITECH_MOMO_WHEEL2: break; default: expand_a = expand_a >> 1; break; } value[0] = 0xfe; value[1] = 0x0d; value[2] = expand_a / 0xaaaa; value[3] = expand_a / 0xaaaa; value[4] = expand_b / 0xaaaa; value[5] = 0x00; value[6] = 0x00; hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); /* Activate Auto-Center */ value[0] = 0x14; value[1] = 0x00; value[2] = 0x00; value[3] = 0x00; value[4] = 0x00; value[5] = 0x00; value[6] = 0x00; hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); spin_unlock_irqrestore(&entry->report_lock, flags); } /* Sends autocentering command compatible with Formula Force EX */ static void lg4ff_set_autocenter_ffex(struct input_dev *dev, u16 magnitude) { struct hid_device *hid = input_get_drvdata(dev); struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; unsigned long flags; s32 *value; magnitude = magnitude * 90 / 65535; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return; } value = entry->report->field[0]->value; spin_lock_irqsave(&entry->report_lock, flags); value[0] = 0xfe; value[1] = 0x03; value[2] = magnitude >> 14; value[3] = magnitude >> 14; value[4] = magnitude; value[5] = 0x00; value[6] = 0x00; hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); spin_unlock_irqrestore(&entry->report_lock, flags); } /* Sends command to set range compatible with G25/G27/Driving Force GT */ static void lg4ff_set_range_g25(struct hid_device *hid, u16 range) { struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; unsigned long flags; s32 *value; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return; } value = entry->report->field[0]->value; dbg_hid("G25/G27/DFGT: setting range to %u\n", range); spin_lock_irqsave(&entry->report_lock, flags); value[0] = 0xf8; value[1] = 0x81; value[2] = range & 0x00ff; value[3] = (range & 0xff00) >> 8; value[4] = 0x00; value[5] = 0x00; value[6] = 0x00; hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); spin_unlock_irqrestore(&entry->report_lock, flags); } /* Sends commands to set range compatible with Driving Force Pro wheel */ static void lg4ff_set_range_dfp(struct hid_device *hid, u16 range) { struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; unsigned long flags; int start_left, start_right, full_range; s32 *value; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return; } value = entry->report->field[0]->value; dbg_hid("Driving Force Pro: setting range to %u\n", range); /* Prepare "coarse" limit command */ spin_lock_irqsave(&entry->report_lock, flags); value[0] = 0xf8; value[1] = 0x00; /* Set later */ value[2] = 0x00; value[3] = 0x00; value[4] = 0x00; value[5] = 0x00; value[6] = 0x00; if (range > 200) { value[1] = 0x03; full_range = 900; } else { value[1] = 0x02; full_range = 200; } hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); /* Prepare "fine" limit command */ value[0] = 0x81; value[1] = 0x0b; value[2] = 0x00; value[3] = 0x00; value[4] = 0x00; value[5] = 0x00; value[6] = 0x00; if (range == 200 || range == 900) { /* Do not apply any fine limit */ hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); spin_unlock_irqrestore(&entry->report_lock, flags); return; } /* Construct fine limit command */ start_left = (((full_range - range + 1) * 2047) / full_range); start_right = 0xfff - start_left; value[2] = start_left >> 4; value[3] = start_right >> 4; value[4] = 0xff; value[5] = (start_right & 0xe) << 4 | (start_left & 0xe); value[6] = 0xff; hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); spin_unlock_irqrestore(&entry->report_lock, flags); } static const struct lg4ff_compat_mode_switch *lg4ff_get_mode_switch_command(const u16 real_product_id, const u16 target_product_id) { switch (real_product_id) { case USB_DEVICE_ID_LOGITECH_DFP_WHEEL: switch (target_product_id) { case USB_DEVICE_ID_LOGITECH_DFP_WHEEL: return &lg4ff_mode_switch_ext01_dfp; /* DFP can only be switched to its native mode */ default: return NULL; } break; case USB_DEVICE_ID_LOGITECH_G25_WHEEL: switch (target_product_id) { case USB_DEVICE_ID_LOGITECH_DFP_WHEEL: return &lg4ff_mode_switch_ext01_dfp; case USB_DEVICE_ID_LOGITECH_G25_WHEEL: return &lg4ff_mode_switch_ext16_g25; /* G25 can only be switched to DFP mode or its native mode */ default: return NULL; } break; case USB_DEVICE_ID_LOGITECH_G27_WHEEL: switch (target_product_id) { case USB_DEVICE_ID_LOGITECH_WHEEL: return &lg4ff_mode_switch_ext09_dfex; case USB_DEVICE_ID_LOGITECH_DFP_WHEEL: return &lg4ff_mode_switch_ext09_dfp; case USB_DEVICE_ID_LOGITECH_G25_WHEEL: return &lg4ff_mode_switch_ext09_g25; case USB_DEVICE_ID_LOGITECH_G27_WHEEL: return &lg4ff_mode_switch_ext09_g27; /* G27 can only be switched to DF-EX, DFP, G25 or its native mode */ default: return NULL; } break; case USB_DEVICE_ID_LOGITECH_G29_WHEEL: switch (target_product_id) { case USB_DEVICE_ID_LOGITECH_DFP_WHEEL: return &lg4ff_mode_switch_ext09_dfp; case USB_DEVICE_ID_LOGITECH_DFGT_WHEEL: return &lg4ff_mode_switch_ext09_dfgt; case USB_DEVICE_ID_LOGITECH_G25_WHEEL: return &lg4ff_mode_switch_ext09_g25; case USB_DEVICE_ID_LOGITECH_G27_WHEEL: return &lg4ff_mode_switch_ext09_g27; case USB_DEVICE_ID_LOGITECH_G29_WHEEL: return &lg4ff_mode_switch_ext09_g29; /* G29 can only be switched to DF-EX, DFP, DFGT, G25, G27 or its native mode */ default: return NULL; } break; case USB_DEVICE_ID_LOGITECH_DFGT_WHEEL: switch (target_product_id) { case USB_DEVICE_ID_LOGITECH_WHEEL: return &lg4ff_mode_switch_ext09_dfex; case USB_DEVICE_ID_LOGITECH_DFP_WHEEL: return &lg4ff_mode_switch_ext09_dfp; case USB_DEVICE_ID_LOGITECH_DFGT_WHEEL: return &lg4ff_mode_switch_ext09_dfgt; /* DFGT can only be switched to DF-EX, DFP or its native mode */ default: return NULL; } break; /* No other wheels have multiple modes */ default: return NULL; } } static int lg4ff_switch_compatibility_mode(struct hid_device *hid, const struct lg4ff_compat_mode_switch *s) { struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; unsigned long flags; s32 *value; u8 i; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return -EINVAL; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return -EINVAL; } value = entry->report->field[0]->value; spin_lock_irqsave(&entry->report_lock, flags); for (i = 0; i < s->cmd_count; i++) { u8 j; for (j = 0; j < 7; j++) value[j] = s->cmd[j + (7*i)]; hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); } spin_unlock_irqrestore(&entry->report_lock, flags); hid_hw_wait(hid); return 0; } static ssize_t lg4ff_alternate_modes_show(struct device *dev, struct device_attribute *attr, char *buf) { struct hid_device *hid = to_hid_device(dev); struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; ssize_t count = 0; int i; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return 0; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return 0; } if (!entry->wdata.real_name) { hid_err(hid, "NULL pointer to string\n"); return 0; } for (i = 0; i < LG4FF_MODE_MAX_IDX; i++) { if (entry->wdata.alternate_modes & BIT(i)) { /* Print tag and full name */ count += sysfs_emit_at(buf, count, "%s: %s", lg4ff_alternate_modes[i].tag, !lg4ff_alternate_modes[i].product_id ? entry->wdata.real_name : lg4ff_alternate_modes[i].name); if (count >= PAGE_SIZE - 1) return count; /* Mark the currently active mode with an asterisk */ if (lg4ff_alternate_modes[i].product_id == entry->wdata.product_id || (lg4ff_alternate_modes[i].product_id == 0 && entry->wdata.product_id == entry->wdata.real_product_id)) count += sysfs_emit_at(buf, count, " *\n"); else count += sysfs_emit_at(buf, count, "\n"); if (count >= PAGE_SIZE - 1) return count; } } return count; } static ssize_t lg4ff_alternate_modes_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct hid_device *hid = to_hid_device(dev); struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; const struct lg4ff_compat_mode_switch *s; u16 target_product_id = 0; int i, ret; char *lbuf; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return -EINVAL; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return -EINVAL; } /* Allow \n at the end of the input parameter */ lbuf = kasprintf(GFP_KERNEL, "%s", buf); if (!lbuf) return -ENOMEM; i = strlen(lbuf); if (i == 0) { kfree(lbuf); return -EINVAL; } if (lbuf[i-1] == '\n') { if (i == 1) { kfree(lbuf); return -EINVAL; } lbuf[i-1] = '\0'; } for (i = 0; i < LG4FF_MODE_MAX_IDX; i++) { const u16 mode_product_id = lg4ff_alternate_modes[i].product_id; const char *tag = lg4ff_alternate_modes[i].tag; if (entry->wdata.alternate_modes & BIT(i)) { if (!strcmp(tag, lbuf)) { if (!mode_product_id) target_product_id = entry->wdata.real_product_id; else target_product_id = mode_product_id; break; } } } if (i == LG4FF_MODE_MAX_IDX) { hid_info(hid, "Requested mode \"%s\" is not supported by the device\n", lbuf); kfree(lbuf); return -EINVAL; } kfree(lbuf); /* Not needed anymore */ if (target_product_id == entry->wdata.product_id) /* Nothing to do */ return count; /* Automatic switching has to be disabled for the switch to DF-EX mode to work correctly */ if (target_product_id == USB_DEVICE_ID_LOGITECH_WHEEL && !lg4ff_no_autoswitch) { hid_info(hid, "\"%s\" cannot be switched to \"DF-EX\" mode. Load the \"hid_logitech\" module with \"lg4ff_no_autoswitch=1\" parameter set and try again\n", entry->wdata.real_name); return -EINVAL; } /* Take care of hardware limitations */ if ((entry->wdata.real_product_id == USB_DEVICE_ID_LOGITECH_DFP_WHEEL || entry->wdata.real_product_id == USB_DEVICE_ID_LOGITECH_G25_WHEEL) && entry->wdata.product_id > target_product_id) { hid_info(hid, "\"%s\" cannot be switched back into \"%s\" mode\n", entry->wdata.real_name, lg4ff_alternate_modes[i].name); return -EINVAL; } s = lg4ff_get_mode_switch_command(entry->wdata.real_product_id, target_product_id); if (!s) { hid_err(hid, "Invalid target product ID %X\n", target_product_id); return -EINVAL; } ret = lg4ff_switch_compatibility_mode(hid, s); return (ret == 0 ? count : ret); } static DEVICE_ATTR(alternate_modes, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH, lg4ff_alternate_modes_show, lg4ff_alternate_modes_store); static ssize_t lg4ff_combine_show(struct device *dev, struct device_attribute *attr, char *buf) { struct hid_device *hid = to_hid_device(dev); struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; size_t count; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return 0; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return 0; } count = sysfs_emit(buf, "%u\n", entry->wdata.combine); return count; } static ssize_t lg4ff_combine_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct hid_device *hid = to_hid_device(dev); struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; u16 combine = simple_strtoul(buf, NULL, 10); drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return -EINVAL; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return -EINVAL; } if (combine > 1) combine = 1; entry->wdata.combine = combine; return count; } static DEVICE_ATTR(combine_pedals, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH, lg4ff_combine_show, lg4ff_combine_store); /* Export the currently set range of the wheel */ static ssize_t lg4ff_range_show(struct device *dev, struct device_attribute *attr, char *buf) { struct hid_device *hid = to_hid_device(dev); struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; size_t count; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return 0; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return 0; } count = sysfs_emit(buf, "%u\n", entry->wdata.range); return count; } /* Set range to user specified value, call appropriate function * according to the type of the wheel */ static ssize_t lg4ff_range_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct hid_device *hid = to_hid_device(dev); struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; u16 range = simple_strtoul(buf, NULL, 10); drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return -EINVAL; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return -EINVAL; } if (range == 0) range = entry->wdata.max_range; /* Check if the wheel supports range setting * and that the range is within limits for the wheel */ if (entry->wdata.set_range && range >= entry->wdata.min_range && range <= entry->wdata.max_range) { entry->wdata.set_range(hid, range); entry->wdata.range = range; } return count; } static DEVICE_ATTR(range, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH, lg4ff_range_show, lg4ff_range_store); static ssize_t lg4ff_real_id_show(struct device *dev, struct device_attribute *attr, char *buf) { struct hid_device *hid = to_hid_device(dev); struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; size_t count; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return 0; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return 0; } if (!entry->wdata.real_tag || !entry->wdata.real_name) { hid_err(hid, "NULL pointer to string\n"); return 0; } count = sysfs_emit(buf, "%s: %s\n", entry->wdata.real_tag, entry->wdata.real_name); return count; } static ssize_t lg4ff_real_id_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { /* Real ID is a read-only value */ return -EPERM; } static DEVICE_ATTR(real_id, S_IRUGO, lg4ff_real_id_show, lg4ff_real_id_store); #ifdef CONFIG_LEDS_CLASS static void lg4ff_set_leds(struct hid_device *hid, u8 leds) { struct lg_drv_data *drv_data; struct lg4ff_device_entry *entry; unsigned long flags; s32 *value; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Private driver data not found!\n"); return; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found!\n"); return; } value = entry->report->field[0]->value; spin_lock_irqsave(&entry->report_lock, flags); value[0] = 0xf8; value[1] = 0x12; value[2] = leds; value[3] = 0x00; value[4] = 0x00; value[5] = 0x00; value[6] = 0x00; hid_hw_request(hid, entry->report, HID_REQ_SET_REPORT); spin_unlock_irqrestore(&entry->report_lock, flags); } static void lg4ff_led_set_brightness(struct led_classdev *led_cdev, enum led_brightness value) { struct device *dev = led_cdev->dev->parent; struct hid_device *hid = to_hid_device(dev); struct lg_drv_data *drv_data = hid_get_drvdata(hid); struct lg4ff_device_entry *entry; int i, state = 0; if (!drv_data) { hid_err(hid, "Device data not found."); return; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found."); return; } for (i = 0; i < 5; i++) { if (led_cdev != entry->wdata.led[i]) continue; state = (entry->wdata.led_state >> i) & 1; if (value == LED_OFF && state) { entry->wdata.led_state &= ~(1 << i); lg4ff_set_leds(hid, entry->wdata.led_state); } else if (value != LED_OFF && !state) { entry->wdata.led_state |= 1 << i; lg4ff_set_leds(hid, entry->wdata.led_state); } break; } } static enum led_brightness lg4ff_led_get_brightness(struct led_classdev *led_cdev) { struct device *dev = led_cdev->dev->parent; struct hid_device *hid = to_hid_device(dev); struct lg_drv_data *drv_data = hid_get_drvdata(hid); struct lg4ff_device_entry *entry; int i, value = 0; if (!drv_data) { hid_err(hid, "Device data not found."); return LED_OFF; } entry = drv_data->device_props; if (!entry) { hid_err(hid, "Device properties not found."); return LED_OFF; } for (i = 0; i < 5; i++) if (led_cdev == entry->wdata.led[i]) { value = (entry->wdata.led_state >> i) & 1; break; } return value ? LED_FULL : LED_OFF; } #endif static u16 lg4ff_identify_multimode_wheel(struct hid_device *hid, const u16 reported_product_id, const u16 bcdDevice) { u32 current_mode; int i; /* identify current mode from USB PID */ for (i = 1; i < ARRAY_SIZE(lg4ff_alternate_modes); i++) { dbg_hid("Testing whether PID is %X\n", lg4ff_alternate_modes[i].product_id); if (reported_product_id == lg4ff_alternate_modes[i].product_id) break; } if (i == ARRAY_SIZE(lg4ff_alternate_modes)) return 0; current_mode = BIT(i); for (i = 0; i < ARRAY_SIZE(lg4ff_main_checklist); i++) { const u16 mask = lg4ff_main_checklist[i]->mask; const u16 result = lg4ff_main_checklist[i]->result; const u16 real_product_id = lg4ff_main_checklist[i]->real_product_id; if ((current_mode & lg4ff_main_checklist[i]->modes) && \ (bcdDevice & mask) == result) { dbg_hid("Found wheel with real PID %X whose reported PID is %X\n", real_product_id, reported_product_id); return real_product_id; } } /* No match found. This is either Driving Force or an unknown * wheel model, do not touch it */ dbg_hid("Wheel with bcdDevice %X was not recognized as multimode wheel, leaving in its current mode\n", bcdDevice); return 0; } static int lg4ff_handle_multimode_wheel(struct hid_device *hid, u16 *real_product_id, const u16 bcdDevice) { const u16 reported_product_id = hid->product; int ret; *real_product_id = lg4ff_identify_multimode_wheel(hid, reported_product_id, bcdDevice); /* Probed wheel is not a multimode wheel */ if (!*real_product_id) { *real_product_id = reported_product_id; dbg_hid("Wheel is not a multimode wheel\n"); return LG4FF_MMODE_NOT_MULTIMODE; } /* Switch from "Driving Force" mode to native mode automatically. * Otherwise keep the wheel in its current mode */ if (reported_product_id == USB_DEVICE_ID_LOGITECH_WHEEL && reported_product_id != *real_product_id && !lg4ff_no_autoswitch) { const struct lg4ff_compat_mode_switch *s = lg4ff_get_mode_switch_command(*real_product_id, *real_product_id); if (!s) { hid_err(hid, "Invalid product id %X\n", *real_product_id); return LG4FF_MMODE_NOT_MULTIMODE; } ret = lg4ff_switch_compatibility_mode(hid, s); if (ret) { /* Wheel could not have been switched to native mode, * leave it in "Driving Force" mode and continue */ hid_err(hid, "Unable to switch wheel mode, errno %d\n", ret); return LG4FF_MMODE_IS_MULTIMODE; } return LG4FF_MMODE_SWITCHED; } return LG4FF_MMODE_IS_MULTIMODE; } int lg4ff_init(struct hid_device *hid) { struct hid_input *hidinput; struct input_dev *dev; struct list_head *report_list = &hid->report_enum[HID_OUTPUT_REPORT].report_list; struct hid_report *report = list_entry(report_list->next, struct hid_report, list); const struct usb_device_descriptor *udesc = &(hid_to_usb_dev(hid)->descriptor); const u16 bcdDevice = le16_to_cpu(udesc->bcdDevice); const struct lg4ff_multimode_wheel *mmode_wheel = NULL; struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; int error, i, j; int mmode_ret, mmode_idx = -1; u16 real_product_id; if (list_empty(&hid->inputs)) { hid_err(hid, "no inputs found\n"); return -ENODEV; } hidinput = list_entry(hid->inputs.next, struct hid_input, list); dev = hidinput->input; /* Check that the report looks ok */ if (!hid_validate_values(hid, HID_OUTPUT_REPORT, 0, 0, 7)) return -1; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Cannot add device, private driver data not allocated\n"); return -1; } entry = kzalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return -ENOMEM; spin_lock_init(&entry->report_lock); entry->report = report; drv_data->device_props = entry; /* Check if a multimode wheel has been connected and * handle it appropriately */ mmode_ret = lg4ff_handle_multimode_wheel(hid, &real_product_id, bcdDevice); /* Wheel has been told to switch to native mode. There is no point in going on * with the initialization as the wheel will do a USB reset when it switches mode */ if (mmode_ret == LG4FF_MMODE_SWITCHED) return 0; else if (mmode_ret < 0) { hid_err(hid, "Unable to switch device mode during initialization, errno %d\n", mmode_ret); error = mmode_ret; goto err_init; } /* Check what wheel has been connected */ for (i = 0; i < ARRAY_SIZE(lg4ff_devices); i++) { if (hid->product == lg4ff_devices[i].product_id) { dbg_hid("Found compatible device, product ID %04X\n", lg4ff_devices[i].product_id); break; } } if (i == ARRAY_SIZE(lg4ff_devices)) { hid_err(hid, "This device is flagged to be handled by the lg4ff module but this module does not know how to handle it. " "Please report this as a bug to LKML, Simon Wood <simon@mungewell.org> or " "Michal Maly <madcatxster@devoid-pointer.net>\n"); error = -1; goto err_init; } if (mmode_ret == LG4FF_MMODE_IS_MULTIMODE) { for (mmode_idx = 0; mmode_idx < ARRAY_SIZE(lg4ff_multimode_wheels); mmode_idx++) { if (real_product_id == lg4ff_multimode_wheels[mmode_idx].product_id) break; } if (mmode_idx == ARRAY_SIZE(lg4ff_multimode_wheels)) { hid_err(hid, "Device product ID %X is not listed as a multimode wheel", real_product_id); error = -1; goto err_init; } } /* Set supported force feedback capabilities */ for (j = 0; lg4ff_devices[i].ff_effects[j] >= 0; j++) set_bit(lg4ff_devices[i].ff_effects[j], dev->ffbit); error = input_ff_create_memless(dev, NULL, lg4ff_play); if (error) goto err_init; /* Initialize device properties */ if (mmode_ret == LG4FF_MMODE_IS_MULTIMODE) { if (WARN_ON(mmode_idx == -1)) return -EINVAL; mmode_wheel = &lg4ff_multimode_wheels[mmode_idx]; } lg4ff_init_wheel_data(&entry->wdata, &lg4ff_devices[i], mmode_wheel, real_product_id); /* Check if autocentering is available and * set the centering force to zero by default */ if (test_bit(FF_AUTOCENTER, dev->ffbit)) { /* Formula Force EX expects different autocentering command */ if ((bcdDevice >> 8) == LG4FF_FFEX_REV_MAJ && (bcdDevice & 0xff) == LG4FF_FFEX_REV_MIN) dev->ff->set_autocenter = lg4ff_set_autocenter_ffex; else dev->ff->set_autocenter = lg4ff_set_autocenter_default; dev->ff->set_autocenter(dev, 0); } /* Create sysfs interface */ error = device_create_file(&hid->dev, &dev_attr_combine_pedals); if (error) hid_warn(hid, "Unable to create sysfs interface for \"combine\", errno %d\n", error); error = device_create_file(&hid->dev, &dev_attr_range); if (error) hid_warn(hid, "Unable to create sysfs interface for \"range\", errno %d\n", error); if (mmode_ret == LG4FF_MMODE_IS_MULTIMODE) { error = device_create_file(&hid->dev, &dev_attr_real_id); if (error) hid_warn(hid, "Unable to create sysfs interface for \"real_id\", errno %d\n", error); error = device_create_file(&hid->dev, &dev_attr_alternate_modes); if (error) hid_warn(hid, "Unable to create sysfs interface for \"alternate_modes\", errno %d\n", error); } dbg_hid("sysfs interface created\n"); /* Set the maximum range to start with */ entry->wdata.range = entry->wdata.max_range; if (entry->wdata.set_range) entry->wdata.set_range(hid, entry->wdata.range); #ifdef CONFIG_LEDS_CLASS /* register led subsystem - G27/G29 only */ entry->wdata.led_state = 0; for (j = 0; j < 5; j++) entry->wdata.led[j] = NULL; if (lg4ff_devices[i].product_id == USB_DEVICE_ID_LOGITECH_G27_WHEEL || lg4ff_devices[i].product_id == USB_DEVICE_ID_LOGITECH_G29_WHEEL) { struct led_classdev *led; size_t name_sz; char *name; lg4ff_set_leds(hid, 0); name_sz = strlen(dev_name(&hid->dev)) + 8; for (j = 0; j < 5; j++) { led = kzalloc(sizeof(struct led_classdev)+name_sz, GFP_KERNEL); if (!led) { hid_err(hid, "can't allocate memory for LED %d\n", j); goto err_leds; } name = (void *)(&led[1]); snprintf(name, name_sz, "%s::RPM%d", dev_name(&hid->dev), j+1); led->name = name; led->brightness = 0; led->max_brightness = 1; led->brightness_get = lg4ff_led_get_brightness; led->brightness_set = lg4ff_led_set_brightness; entry->wdata.led[j] = led; error = led_classdev_register(&hid->dev, led); if (error) { hid_err(hid, "failed to register LED %d. Aborting.\n", j); err_leds: /* Deregister LEDs (if any) */ for (j = 0; j < 5; j++) { led = entry->wdata.led[j]; entry->wdata.led[j] = NULL; if (!led) continue; led_classdev_unregister(led); kfree(led); } goto out; /* Let the driver continue without LEDs */ } } } out: #endif hid_info(hid, "Force feedback support for Logitech Gaming Wheels\n"); return 0; err_init: drv_data->device_props = NULL; kfree(entry); return error; } int lg4ff_deinit(struct hid_device *hid) { struct lg4ff_device_entry *entry; struct lg_drv_data *drv_data; drv_data = hid_get_drvdata(hid); if (!drv_data) { hid_err(hid, "Error while deinitializing device, no private driver data.\n"); return -1; } entry = drv_data->device_props; if (!entry) goto out; /* Nothing more to do */ /* Multimode devices will have at least the "MODE_NATIVE" bit set */ if (entry->wdata.alternate_modes) { device_remove_file(&hid->dev, &dev_attr_real_id); device_remove_file(&hid->dev, &dev_attr_alternate_modes); } device_remove_file(&hid->dev, &dev_attr_combine_pedals); device_remove_file(&hid->dev, &dev_attr_range); #ifdef CONFIG_LEDS_CLASS { int j; struct led_classdev *led; /* Deregister LEDs (if any) */ for (j = 0; j < 5; j++) { led = entry->wdata.led[j]; entry->wdata.led[j] = NULL; if (!led) continue; led_classdev_unregister(led); kfree(led); } } #endif drv_data->device_props = NULL; kfree(entry); out: dbg_hid("Device successfully unregistered\n"); return 0; } |
| 65 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | /* SPDX-License-Identifier: GPL-2.0-or-later */ /* * ALSA sequencer Memory Manager * Copyright (c) 1998 by Frank van de Pol <fvdpol@coil.demon.nl> */ #ifndef __SND_SEQ_MEMORYMGR_H #define __SND_SEQ_MEMORYMGR_H #include <sound/seq_kernel.h> #include <linux/poll.h> struct snd_info_buffer; /* aliasing for legacy and UMP event packet handling */ union __snd_seq_event { struct snd_seq_event legacy; #if IS_ENABLED(CONFIG_SND_SEQ_UMP) struct snd_seq_ump_event ump; #endif struct { struct snd_seq_event event; #if IS_ENABLED(CONFIG_SND_SEQ_UMP) u32 extra; #endif } __packed raw; }; /* container for sequencer event (internal use) */ struct snd_seq_event_cell { union { struct snd_seq_event event; union __snd_seq_event ump; }; struct snd_seq_pool *pool; /* used pool */ struct snd_seq_event_cell *next; /* next cell */ }; /* design note: the pool is a contiguous block of memory, if we dynamicly want to add additional cells to the pool be better store this in another pool as we need to know the base address of the pool when releasing memory. */ struct snd_seq_pool { struct snd_seq_event_cell *ptr; /* pointer to first event chunk */ struct snd_seq_event_cell *free; /* pointer to the head of the free list */ int total_elements; /* pool size actually allocated */ atomic_t counter; /* cells free */ int size; /* pool size to be allocated */ int room; /* watermark for sleep/wakeup */ int closing; /* statistics */ int max_used; int event_alloc_nopool; int event_alloc_failures; int event_alloc_success; /* Write locking */ wait_queue_head_t output_sleep; /* Pool lock */ spinlock_t lock; }; void snd_seq_cell_free(struct snd_seq_event_cell *cell); int snd_seq_event_dup(struct snd_seq_pool *pool, struct snd_seq_event *event, struct snd_seq_event_cell **cellp, int nonblock, struct file *file, struct mutex *mutexp); /* return number of unused (free) cells */ static inline int snd_seq_unused_cells(struct snd_seq_pool *pool) { return pool ? pool->total_elements - atomic_read(&pool->counter) : 0; } /* return total number of allocated cells */ static inline int snd_seq_total_cells(struct snd_seq_pool *pool) { return pool ? pool->total_elements : 0; } /* init pool - allocate events */ int snd_seq_pool_init(struct snd_seq_pool *pool); /* done pool - free events */ void snd_seq_pool_mark_closing(struct snd_seq_pool *pool); int snd_seq_pool_done(struct snd_seq_pool *pool); /* create pool */ struct snd_seq_pool *snd_seq_pool_new(int poolsize); /* remove pool */ int snd_seq_pool_delete(struct snd_seq_pool **pool); /* polling */ int snd_seq_pool_poll_wait(struct snd_seq_pool *pool, struct file *file, poll_table *wait); void snd_seq_info_pool(struct snd_info_buffer *buffer, struct snd_seq_pool *pool, char *space); #endif |
| 9 9 9 9 1 1 1 2 2 2 5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | /* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2008, Intel Corporation. * * Author: Alexander Duyck <alexander.h.duyck@intel.com> */ #ifndef __NET_TC_SKBEDIT_H #define __NET_TC_SKBEDIT_H #include <net/act_api.h> #include <linux/tc_act/tc_skbedit.h> struct tcf_skbedit_params { int action; u32 flags; u32 priority; u32 mark; u32 mask; u16 queue_mapping; u16 mapping_mod; u16 ptype; struct rcu_head rcu; }; struct tcf_skbedit { struct tc_action common; struct tcf_skbedit_params __rcu *params; }; #define to_skbedit(a) ((struct tcf_skbedit *)a) /* Return true iff action is the one identified by FLAG. */ static inline bool is_tcf_skbedit_with_flag(const struct tc_action *a, u32 flag) { #ifdef CONFIG_NET_CLS_ACT u32 flags; if (a->ops && a->ops->id == TCA_ID_SKBEDIT) { rcu_read_lock(); flags = rcu_dereference(to_skbedit(a)->params)->flags; rcu_read_unlock(); return flags == flag; } #endif return false; } /* Return true iff action is mark */ static inline bool is_tcf_skbedit_mark(const struct tc_action *a) { return is_tcf_skbedit_with_flag(a, SKBEDIT_F_MARK); } static inline u32 tcf_skbedit_mark(const struct tc_action *a) { u32 mark; rcu_read_lock(); mark = rcu_dereference(to_skbedit(a)->params)->mark; rcu_read_unlock(); return mark; } /* Return true iff action is ptype */ static inline bool is_tcf_skbedit_ptype(const struct tc_action *a) { return is_tcf_skbedit_with_flag(a, SKBEDIT_F_PTYPE); } static inline u32 tcf_skbedit_ptype(const struct tc_action *a) { u16 ptype; rcu_read_lock(); ptype = rcu_dereference(to_skbedit(a)->params)->ptype; rcu_read_unlock(); return ptype; } /* Return true iff action is priority */ static inline bool is_tcf_skbedit_priority(const struct tc_action *a) { return is_tcf_skbedit_with_flag(a, SKBEDIT_F_PRIORITY); } static inline u32 tcf_skbedit_priority(const struct tc_action *a) { u32 priority; rcu_read_lock(); priority = rcu_dereference(to_skbedit(a)->params)->priority; rcu_read_unlock(); return priority; } static inline u16 tcf_skbedit_rx_queue_mapping(const struct tc_action *a) { u16 rx_queue; rcu_read_lock(); rx_queue = rcu_dereference(to_skbedit(a)->params)->queue_mapping; rcu_read_unlock(); return rx_queue; } /* Return true iff action is queue_mapping */ static inline bool is_tcf_skbedit_queue_mapping(const struct tc_action *a) { return is_tcf_skbedit_with_flag(a, SKBEDIT_F_QUEUE_MAPPING); } /* Return true if action is on ingress traffic */ static inline bool is_tcf_skbedit_ingress(u32 flags) { return flags & TCA_ACT_FLAGS_AT_INGRESS; } static inline bool is_tcf_skbedit_tx_queue_mapping(const struct tc_action *a) { return is_tcf_skbedit_queue_mapping(a) && !is_tcf_skbedit_ingress(a->tcfa_flags); } static inline bool is_tcf_skbedit_rx_queue_mapping(const struct tc_action *a) { return is_tcf_skbedit_queue_mapping(a) && is_tcf_skbedit_ingress(a->tcfa_flags); } /* Return true iff action is inheritdsfield */ static inline bool is_tcf_skbedit_inheritdsfield(const struct tc_action *a) { return is_tcf_skbedit_with_flag(a, SKBEDIT_F_INHERITDSFIELD); } #endif /* __NET_TC_SKBEDIT_H */ |
| 2 26 26 3 33 33 33 33 34 26 33 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 | // SPDX-License-Identifier: GPL-2.0-only #include <linux/kernel.h> #include <linux/module.h> #include <linux/netfilter.h> #include <linux/rhashtable.h> #include <linux/netdevice.h> #include <net/flow_offload.h> #include <net/netfilter/nf_flow_table.h> struct flow_offload_xdp_ft { struct list_head head; struct nf_flowtable *ft; struct rcu_head rcuhead; }; struct flow_offload_xdp { struct hlist_node hnode; unsigned long net_device_addr; struct list_head head; }; #define NF_XDP_HT_BITS 4 static DEFINE_HASHTABLE(nf_xdp_hashtable, NF_XDP_HT_BITS); static DEFINE_MUTEX(nf_xdp_hashtable_lock); /* caller must hold rcu read lock */ struct nf_flowtable *nf_flowtable_by_dev(const struct net_device *dev) { unsigned long key = (unsigned long)dev; struct flow_offload_xdp *iter; hash_for_each_possible_rcu(nf_xdp_hashtable, iter, hnode, key) { if (key == iter->net_device_addr) { struct flow_offload_xdp_ft *ft_elem; /* The user is supposed to insert a given net_device * just into a single nf_flowtable so we always return * the first element here. */ ft_elem = list_first_or_null_rcu(&iter->head, struct flow_offload_xdp_ft, head); return ft_elem ? ft_elem->ft : NULL; } } return NULL; } static int nf_flowtable_by_dev_insert(struct nf_flowtable *ft, const struct net_device *dev) { struct flow_offload_xdp *iter, *elem = NULL; unsigned long key = (unsigned long)dev; struct flow_offload_xdp_ft *ft_elem; ft_elem = kzalloc(sizeof(*ft_elem), GFP_KERNEL_ACCOUNT); if (!ft_elem) return -ENOMEM; ft_elem->ft = ft; mutex_lock(&nf_xdp_hashtable_lock); hash_for_each_possible(nf_xdp_hashtable, iter, hnode, key) { if (key == iter->net_device_addr) { elem = iter; break; } } if (!elem) { elem = kzalloc(sizeof(*elem), GFP_KERNEL_ACCOUNT); if (!elem) goto err_unlock; elem->net_device_addr = key; INIT_LIST_HEAD(&elem->head); hash_add_rcu(nf_xdp_hashtable, &elem->hnode, key); } list_add_tail_rcu(&ft_elem->head, &elem->head); mutex_unlock(&nf_xdp_hashtable_lock); return 0; err_unlock: mutex_unlock(&nf_xdp_hashtable_lock); kfree(ft_elem); return -ENOMEM; } static void nf_flowtable_by_dev_remove(struct nf_flowtable *ft, const struct net_device *dev) { struct flow_offload_xdp *iter, *elem = NULL; unsigned long key = (unsigned long)dev; mutex_lock(&nf_xdp_hashtable_lock); hash_for_each_possible(nf_xdp_hashtable, iter, hnode, key) { if (key == iter->net_device_addr) { elem = iter; break; } } if (elem) { struct flow_offload_xdp_ft *ft_elem, *ft_next; list_for_each_entry_safe(ft_elem, ft_next, &elem->head, head) { if (ft_elem->ft == ft) { list_del_rcu(&ft_elem->head); kfree_rcu(ft_elem, rcuhead); } } if (list_empty(&elem->head)) hash_del_rcu(&elem->hnode); else elem = NULL; } mutex_unlock(&nf_xdp_hashtable_lock); if (elem) { synchronize_rcu(); kfree(elem); } } int nf_flow_offload_xdp_setup(struct nf_flowtable *flowtable, struct net_device *dev, enum flow_block_command cmd) { switch (cmd) { case FLOW_BLOCK_BIND: return nf_flowtable_by_dev_insert(flowtable, dev); case FLOW_BLOCK_UNBIND: nf_flowtable_by_dev_remove(flowtable, dev); return 0; } WARN_ON_ONCE(1); return 0; } |
| 5 3 3 3 1 2 1 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | // SPDX-License-Identifier: GPL-2.0 /* MPTCP socket monitoring support * * Copyright (c) 2019 Red Hat * * Author: Davide Caratti <dcaratti@redhat.com> */ #include <linux/kernel.h> #include <linux/net.h> #include <linux/inet_diag.h> #include <net/netlink.h> #include "protocol.h" static int subflow_get_info(struct sock *sk, struct sk_buff *skb, bool net_admin) { struct mptcp_subflow_context *sf; struct nlattr *start; u32 flags = 0; bool slow; int err; if (inet_sk_state_load(sk) == TCP_LISTEN) return 0; start = nla_nest_start_noflag(skb, INET_ULP_INFO_MPTCP); if (!start) return -EMSGSIZE; slow = lock_sock_fast(sk); rcu_read_lock(); sf = rcu_dereference(inet_csk(sk)->icsk_ulp_data); if (!sf) { err = 0; goto nla_failure; } if (sf->mp_capable) flags |= MPTCP_SUBFLOW_FLAG_MCAP_REM; if (sf->request_mptcp) flags |= MPTCP_SUBFLOW_FLAG_MCAP_LOC; if (sf->mp_join) flags |= MPTCP_SUBFLOW_FLAG_JOIN_REM; if (sf->request_join) flags |= MPTCP_SUBFLOW_FLAG_JOIN_LOC; if (sf->backup) flags |= MPTCP_SUBFLOW_FLAG_BKUP_REM; if (sf->request_bkup) flags |= MPTCP_SUBFLOW_FLAG_BKUP_LOC; if (READ_ONCE(sf->fully_established)) flags |= MPTCP_SUBFLOW_FLAG_FULLY_ESTABLISHED; if (sf->conn_finished) flags |= MPTCP_SUBFLOW_FLAG_CONNECTED; if (sf->map_valid) flags |= MPTCP_SUBFLOW_FLAG_MAPVALID; if (nla_put_u32(skb, MPTCP_SUBFLOW_ATTR_TOKEN_REM, sf->remote_token) || nla_put_u32(skb, MPTCP_SUBFLOW_ATTR_TOKEN_LOC, sf->token) || nla_put_u32(skb, MPTCP_SUBFLOW_ATTR_FLAGS, flags) || nla_put_u8(skb, MPTCP_SUBFLOW_ATTR_ID_REM, sf->remote_id) || nla_put_u8(skb, MPTCP_SUBFLOW_ATTR_ID_LOC, subflow_get_local_id(sf))) { err = -EMSGSIZE; goto nla_failure; } /* Only export seq related counters to user with CAP_NET_ADMIN */ if (net_admin && (nla_put_u32(skb, MPTCP_SUBFLOW_ATTR_RELWRITE_SEQ, sf->rel_write_seq) || nla_put_u64_64bit(skb, MPTCP_SUBFLOW_ATTR_MAP_SEQ, sf->map_seq, MPTCP_SUBFLOW_ATTR_PAD) || nla_put_u32(skb, MPTCP_SUBFLOW_ATTR_MAP_SFSEQ, sf->map_subflow_seq) || nla_put_u32(skb, MPTCP_SUBFLOW_ATTR_SSN_OFFSET, sf->ssn_offset) || nla_put_u16(skb, MPTCP_SUBFLOW_ATTR_MAP_DATALEN, sf->map_data_len))) { err = -EMSGSIZE; goto nla_failure; } rcu_read_unlock(); unlock_sock_fast(sk, slow); nla_nest_end(skb, start); return 0; nla_failure: rcu_read_unlock(); unlock_sock_fast(sk, slow); nla_nest_cancel(skb, start); return err; } static size_t subflow_get_info_size(const struct sock *sk, bool net_admin) { size_t size = 0; size += nla_total_size(0) + /* INET_ULP_INFO_MPTCP */ nla_total_size(4) + /* MPTCP_SUBFLOW_ATTR_TOKEN_REM */ nla_total_size(4) + /* MPTCP_SUBFLOW_ATTR_TOKEN_LOC */ nla_total_size(4) + /* MPTCP_SUBFLOW_ATTR_FLAGS */ nla_total_size(1) + /* MPTCP_SUBFLOW_ATTR_ID_REM */ nla_total_size(1) + /* MPTCP_SUBFLOW_ATTR_ID_LOC */ 0; if (net_admin) size += nla_total_size(4) + /* MPTCP_SUBFLOW_ATTR_RELWRITE_SEQ */ nla_total_size_64bit(8) + /* MPTCP_SUBFLOW_ATTR_MAP_SEQ */ nla_total_size(4) + /* MPTCP_SUBFLOW_ATTR_MAP_SFSEQ */ nla_total_size(4) + /* MPTCP_SUBFLOW_ATTR_SSN_OFFSET */ nla_total_size(2) + /* MPTCP_SUBFLOW_ATTR_MAP_DATALEN */ 0; return size; } void mptcp_diag_subflow_init(struct tcp_ulp_ops *ops) { ops->get_info = subflow_get_info; ops->get_info_size = subflow_get_info_size; } |
| 10 10 10 10 10 6 1 1 3 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 | // SPDX-License-Identifier: GPL-2.0-only #include <linux/kernel.h> #include <linux/init.h> #include <linux/module.h> #include <linux/netlink.h> #include <linux/netfilter.h> #include <linux/netfilter/nf_tables.h> #include <net/netfilter/nf_tables_core.h> #include <net/netfilter/nf_tables.h> #include <net/netfilter/nft_fib.h> #include <net/flow.h> #include <net/ip.h> #include <net/ip_fib.h> #include <net/route.h> /* don't try to find route from mcast/bcast/zeronet */ static __be32 get_saddr(__be32 addr) { if (ipv4_is_multicast(addr) || ipv4_is_lbcast(addr) || ipv4_is_zeronet(addr)) return 0; return addr; } void nft_fib4_eval_type(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt) { const struct nft_fib *priv = nft_expr_priv(expr); int noff = skb_network_offset(pkt->skb); u32 *dst = ®s->data[priv->dreg]; const struct net_device *dev = NULL; struct iphdr *iph, _iph; __be32 addr; if (priv->flags & NFTA_FIB_F_IIF) dev = nft_in(pkt); else if (priv->flags & NFTA_FIB_F_OIF) dev = nft_out(pkt); iph = skb_header_pointer(pkt->skb, noff, sizeof(_iph), &_iph); if (!iph) { regs->verdict.code = NFT_BREAK; return; } if (priv->flags & NFTA_FIB_F_DADDR) addr = iph->daddr; else addr = iph->saddr; if (priv->flags & (NFTA_FIB_F_IIF | NFTA_FIB_F_OIF)) { *dst = inet_dev_addr_type(nft_net(pkt), dev, addr); return; } *dst = inet_addr_type_dev_table(nft_net(pkt), pkt->skb->dev, addr); } EXPORT_SYMBOL_GPL(nft_fib4_eval_type); void nft_fib4_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt) { const struct nft_fib *priv = nft_expr_priv(expr); int noff = skb_network_offset(pkt->skb); u32 *dest = ®s->data[priv->dreg]; struct iphdr *iph, _iph; struct fib_result res; struct flowi4 fl4 = { .flowi4_scope = RT_SCOPE_UNIVERSE, .flowi4_iif = LOOPBACK_IFINDEX, .flowi4_proto = pkt->tprot, .flowi4_uid = sock_net_uid(nft_net(pkt), NULL), }; const struct net_device *oif; const struct net_device *found; if (nft_fib_can_skip(pkt)) { nft_fib_store_result(dest, priv, nft_in(pkt)); return; } /* * Do not set flowi4_oif, it restricts results (for example, asking * for oif 3 will get RTN_UNICAST result even if the daddr exits * on another interface. * * Search results for the desired outinterface instead. */ if (priv->flags & NFTA_FIB_F_OIF) oif = nft_out(pkt); else if (priv->flags & NFTA_FIB_F_IIF) oif = nft_in(pkt); else oif = NULL; fl4.flowi4_l3mdev = nft_fib_l3mdev_master_ifindex_rcu(pkt, oif); iph = skb_header_pointer(pkt->skb, noff, sizeof(_iph), &_iph); if (!iph) { regs->verdict.code = NFT_BREAK; return; } if (ipv4_is_zeronet(iph->saddr)) { if (ipv4_is_lbcast(iph->daddr) || ipv4_is_local_multicast(iph->daddr)) { nft_fib_store_result(dest, priv, pkt->skb->dev); return; } } if (priv->flags & NFTA_FIB_F_MARK) fl4.flowi4_mark = pkt->skb->mark; fl4.flowi4_dscp = ip4h_dscp(iph); if (priv->flags & NFTA_FIB_F_DADDR) { fl4.daddr = iph->daddr; fl4.saddr = get_saddr(iph->saddr); } else { if (nft_hook(pkt) == NF_INET_FORWARD && priv->flags & NFTA_FIB_F_IIF) fl4.flowi4_iif = nft_out(pkt)->ifindex; fl4.daddr = iph->saddr; fl4.saddr = get_saddr(iph->daddr); } *dest = 0; if (fib_lookup(nft_net(pkt), &fl4, &res, FIB_LOOKUP_IGNORE_LINKSTATE)) return; switch (res.type) { case RTN_UNICAST: break; case RTN_LOCAL: /* Should not see RTN_LOCAL here */ return; default: break; } if (!oif) { found = FIB_RES_DEV(res); } else { if (!fib_info_nh_uses_dev(res.fi, oif)) return; found = oif; } nft_fib_store_result(dest, priv, found); } EXPORT_SYMBOL_GPL(nft_fib4_eval); static struct nft_expr_type nft_fib4_type; static const struct nft_expr_ops nft_fib4_type_ops = { .type = &nft_fib4_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_fib)), .eval = nft_fib4_eval_type, .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, .reduce = nft_fib_reduce, }; static const struct nft_expr_ops nft_fib4_ops = { .type = &nft_fib4_type, .size = NFT_EXPR_SIZE(sizeof(struct nft_fib)), .eval = nft_fib4_eval, .init = nft_fib_init, .dump = nft_fib_dump, .validate = nft_fib_validate, .reduce = nft_fib_reduce, }; static const struct nft_expr_ops * nft_fib4_select_ops(const struct nft_ctx *ctx, const struct nlattr * const tb[]) { enum nft_fib_result result; if (!tb[NFTA_FIB_RESULT]) return ERR_PTR(-EINVAL); result = ntohl(nla_get_be32(tb[NFTA_FIB_RESULT])); switch (result) { case NFT_FIB_RESULT_OIF: return &nft_fib4_ops; case NFT_FIB_RESULT_OIFNAME: return &nft_fib4_ops; case NFT_FIB_RESULT_ADDRTYPE: return &nft_fib4_type_ops; default: return ERR_PTR(-EOPNOTSUPP); } } static struct nft_expr_type nft_fib4_type __read_mostly = { .name = "fib", .select_ops = nft_fib4_select_ops, .policy = nft_fib_policy, .maxattr = NFTA_FIB_MAX, .family = NFPROTO_IPV4, .owner = THIS_MODULE, }; static int __init nft_fib4_module_init(void) { return nft_register_expr(&nft_fib4_type); } static void __exit nft_fib4_module_exit(void) { nft_unregister_expr(&nft_fib4_type); } module_init(nft_fib4_module_init); module_exit(nft_fib4_module_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Florian Westphal <fw@strlen.de>"); MODULE_ALIAS_NFT_AF_EXPR(2, "fib"); MODULE_DESCRIPTION("nftables fib / ip route lookup support"); |
| 7 7 7 7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 | // SPDX-License-Identifier: MIT /* * Copyright © 2021 Intel Corporation */ #include <drm/drm_edid.h> #include <drm/drm_print.h> #include "drm_crtc_internal.h" #include "drm_displayid_internal.h" static const struct displayid_header * displayid_get_header(const u8 *displayid, int length, int index) { const struct displayid_header *base; if (sizeof(*base) > length - index) return ERR_PTR(-EINVAL); base = (const struct displayid_header *)&displayid[index]; return base; } static const struct displayid_header * validate_displayid(const u8 *displayid, int length, int idx) { int i, dispid_length; u8 csum = 0; const struct displayid_header *base; base = displayid_get_header(displayid, length, idx); if (IS_ERR(base)) return base; /* +1 for DispID checksum */ dispid_length = sizeof(*base) + base->bytes + 1; if (dispid_length > length - idx) return ERR_PTR(-EINVAL); for (i = 0; i < dispid_length; i++) csum += displayid[idx + i]; if (csum) { DRM_NOTE("DisplayID checksum invalid, remainder is %d\n", csum); return ERR_PTR(-EINVAL); } return base; } static const u8 *drm_find_displayid_extension(const struct drm_edid *drm_edid, int *length, int *idx, int *ext_index) { const struct displayid_header *base; const u8 *displayid; displayid = drm_edid_find_extension(drm_edid, DISPLAYID_EXT, ext_index); if (!displayid) return NULL; /* EDID extensions block checksum isn't for us */ *length = EDID_LENGTH - 1; *idx = 1; base = validate_displayid(displayid, *length, *idx); if (IS_ERR(base)) return NULL; *length = *idx + sizeof(*base) + base->bytes; return displayid; } void displayid_iter_edid_begin(const struct drm_edid *drm_edid, struct displayid_iter *iter) { memset(iter, 0, sizeof(*iter)); iter->drm_edid = drm_edid; } static const struct displayid_block * displayid_iter_block(const struct displayid_iter *iter) { const struct displayid_block *block; if (!iter->section) return NULL; block = (const struct displayid_block *)&iter->section[iter->idx]; if (iter->idx + sizeof(*block) <= iter->length && iter->idx + sizeof(*block) + block->num_bytes <= iter->length) return block; return NULL; } const struct displayid_block * __displayid_iter_next(struct displayid_iter *iter) { const struct displayid_block *block; if (!iter->drm_edid) return NULL; if (iter->section) { /* current block should always be valid */ block = displayid_iter_block(iter); if (WARN_ON(!block)) { iter->section = NULL; iter->drm_edid = NULL; return NULL; } /* next block in section */ iter->idx += sizeof(*block) + block->num_bytes; block = displayid_iter_block(iter); if (block) return block; } for (;;) { /* The first section we encounter is the base section */ bool base_section = !iter->section; iter->section = drm_find_displayid_extension(iter->drm_edid, &iter->length, &iter->idx, &iter->ext_index); if (!iter->section) { iter->drm_edid = NULL; return NULL; } /* Save the structure version and primary use case. */ if (base_section) { const struct displayid_header *base; base = displayid_get_header(iter->section, iter->length, iter->idx); if (!IS_ERR(base)) { iter->version = base->rev; iter->primary_use = base->prod_id; } } iter->idx += sizeof(struct displayid_header); block = displayid_iter_block(iter); if (block) return block; } } void displayid_iter_end(struct displayid_iter *iter) { memset(iter, 0, sizeof(*iter)); } /* DisplayID Structure Version/Revision from the Base Section. */ u8 displayid_version(const struct displayid_iter *iter) { return iter->version; } /* * DisplayID Primary Use Case (2.0+) or Product Type Identifier (1.0-1.3) from * the Base Section. */ u8 displayid_primary_use(const struct displayid_iter *iter) { return iter->primary_use; } |
| 27 953 880 74 40 840 1 72 1092 952 956 623 931 11 671 680 654 6 659 5 125 637 29 635 29 635 30 727 504 421 421 422 422 878 873 835 82 68 23 83 61 58 32 353 1008 1010 67 878 43 1067 1 766 1073 48 256 18 261 55 1 5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 | #ifndef IOU_CORE_H #define IOU_CORE_H #include <linux/errno.h> #include <linux/lockdep.h> #include <linux/resume_user_mode.h> #include <linux/kasan.h> #include <linux/poll.h> #include <linux/io_uring_types.h> #include <uapi/linux/eventpoll.h> #include "alloc_cache.h" #include "io-wq.h" #include "slist.h" #include "opdef.h" #ifndef CREATE_TRACE_POINTS #include <trace/events/io_uring.h> #endif #define IORING_FEAT_FLAGS (IORING_FEAT_SINGLE_MMAP |\ IORING_FEAT_NODROP |\ IORING_FEAT_SUBMIT_STABLE |\ IORING_FEAT_RW_CUR_POS |\ IORING_FEAT_CUR_PERSONALITY |\ IORING_FEAT_FAST_POLL |\ IORING_FEAT_POLL_32BITS |\ IORING_FEAT_SQPOLL_NONFIXED |\ IORING_FEAT_EXT_ARG |\ IORING_FEAT_NATIVE_WORKERS |\ IORING_FEAT_RSRC_TAGS |\ IORING_FEAT_CQE_SKIP |\ IORING_FEAT_LINKED_FILE |\ IORING_FEAT_REG_REG_RING |\ IORING_FEAT_RECVSEND_BUNDLE |\ IORING_FEAT_MIN_TIMEOUT |\ IORING_FEAT_RW_ATTR |\ IORING_FEAT_NO_IOWAIT) #define IORING_SETUP_FLAGS (IORING_SETUP_IOPOLL |\ IORING_SETUP_SQPOLL |\ IORING_SETUP_SQ_AFF |\ IORING_SETUP_CQSIZE |\ IORING_SETUP_CLAMP |\ IORING_SETUP_ATTACH_WQ |\ IORING_SETUP_R_DISABLED |\ IORING_SETUP_SUBMIT_ALL |\ IORING_SETUP_COOP_TASKRUN |\ IORING_SETUP_TASKRUN_FLAG |\ IORING_SETUP_SQE128 |\ IORING_SETUP_CQE32 |\ IORING_SETUP_SINGLE_ISSUER |\ IORING_SETUP_DEFER_TASKRUN |\ IORING_SETUP_NO_MMAP |\ IORING_SETUP_REGISTERED_FD_ONLY |\ IORING_SETUP_NO_SQARRAY |\ IORING_SETUP_HYBRID_IOPOLL |\ IORING_SETUP_CQE_MIXED) #define IORING_ENTER_FLAGS (IORING_ENTER_GETEVENTS |\ IORING_ENTER_SQ_WAKEUP |\ IORING_ENTER_SQ_WAIT |\ IORING_ENTER_EXT_ARG |\ IORING_ENTER_REGISTERED_RING |\ IORING_ENTER_ABS_TIMER |\ IORING_ENTER_EXT_ARG_REG |\ IORING_ENTER_NO_IOWAIT) #define SQE_VALID_FLAGS (IOSQE_FIXED_FILE |\ IOSQE_IO_DRAIN |\ IOSQE_IO_LINK |\ IOSQE_IO_HARDLINK |\ IOSQE_ASYNC |\ IOSQE_BUFFER_SELECT |\ IOSQE_CQE_SKIP_SUCCESS) enum { IOU_COMPLETE = 0, IOU_ISSUE_SKIP_COMPLETE = -EIOCBQUEUED, /* * The request has more work to do and should be retried. io_uring will * attempt to wait on the file for eligible opcodes, but otherwise * it'll be handed to iowq for blocking execution. It works for normal * requests as well as for the multi shot mode. */ IOU_RETRY = -EAGAIN, /* * Requeue the task_work to restart operations on this request. The * actual value isn't important, should just be not an otherwise * valid error code, yet less than -MAX_ERRNO and valid internally. */ IOU_REQUEUE = -3072, }; struct io_wait_queue { struct wait_queue_entry wq; struct io_ring_ctx *ctx; unsigned cq_tail; unsigned cq_min_tail; unsigned nr_timeouts; int hit_timeout; ktime_t min_timeout; ktime_t timeout; struct hrtimer t; #ifdef CONFIG_NET_RX_BUSY_POLL ktime_t napi_busy_poll_dt; bool napi_prefer_busy_poll; #endif }; static inline bool io_should_wake(struct io_wait_queue *iowq) { struct io_ring_ctx *ctx = iowq->ctx; int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail; /* * Wake up if we have enough events, or if a timeout occurred since we * started waiting. For timeouts, we always want to return to userspace, * regardless of event count. */ return dist >= 0 || atomic_read(&ctx->cq_timeouts) != iowq->nr_timeouts; } #define IORING_MAX_ENTRIES 32768 #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) unsigned long rings_size(unsigned int flags, unsigned int sq_entries, unsigned int cq_entries, size_t *sq_offset); int io_uring_fill_params(unsigned entries, struct io_uring_params *p); bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32); int io_run_task_work_sig(struct io_ring_ctx *ctx); void io_req_defer_failed(struct io_kiocb *req, s32 res); bool io_post_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); void io_add_aux_cqe(struct io_ring_ctx *ctx, u64 user_data, s32 res, u32 cflags); bool io_req_post_cqe(struct io_kiocb *req, s32 res, u32 cflags); bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe src_cqe[2]); void __io_commit_cqring_flush(struct io_ring_ctx *ctx); void io_req_track_inflight(struct io_kiocb *req); struct file *io_file_get_normal(struct io_kiocb *req, int fd); struct file *io_file_get_fixed(struct io_kiocb *req, int fd, unsigned issue_flags); void __io_req_task_work_add(struct io_kiocb *req, unsigned flags); void io_req_task_work_add_remote(struct io_kiocb *req, unsigned flags); void io_req_task_queue(struct io_kiocb *req); void io_req_task_complete(struct io_kiocb *req, io_tw_token_t tw); void io_req_task_queue_fail(struct io_kiocb *req, int ret); void io_req_task_submit(struct io_kiocb *req, io_tw_token_t tw); struct llist_node *io_handle_tw_list(struct llist_node *node, unsigned int *count, unsigned int max_entries); struct llist_node *tctx_task_work_run(struct io_uring_task *tctx, unsigned int max_entries, unsigned int *count); void tctx_task_work(struct callback_head *cb); __cold void io_uring_cancel_generic(bool cancel_all, struct io_sq_data *sqd); int io_ring_add_registered_file(struct io_uring_task *tctx, struct file *file, int start, int end); void io_req_queue_iowq(struct io_kiocb *req); int io_poll_issue(struct io_kiocb *req, io_tw_token_t tw); int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr); int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin); void __io_submit_flush_completions(struct io_ring_ctx *ctx); struct io_wq_work *io_wq_free_work(struct io_wq_work *work); void io_wq_submit_work(struct io_wq_work *work); void io_free_req(struct io_kiocb *req); void io_queue_next(struct io_kiocb *req); void io_task_refs_refill(struct io_uring_task *tctx); bool __io_alloc_req_refill(struct io_ring_ctx *ctx); bool io_match_task_safe(struct io_kiocb *head, struct io_uring_task *tctx, bool cancel_all); void io_activate_pollwq(struct io_ring_ctx *ctx); static inline void io_lockdep_assert_cq_locked(struct io_ring_ctx *ctx) { #if defined(CONFIG_PROVE_LOCKING) lockdep_assert(in_task()); if (ctx->flags & IORING_SETUP_DEFER_TASKRUN) lockdep_assert_held(&ctx->uring_lock); if (ctx->flags & IORING_SETUP_IOPOLL) { lockdep_assert_held(&ctx->uring_lock); } else if (!ctx->task_complete) { lockdep_assert_held(&ctx->completion_lock); } else if (ctx->submitter_task) { /* * ->submitter_task may be NULL and we can still post a CQE, * if the ring has been setup with IORING_SETUP_R_DISABLED. * Not from an SQE, as those cannot be submitted, but via * updating tagged resources. */ if (!percpu_ref_is_dying(&ctx->refs)) lockdep_assert(current == ctx->submitter_task); } #endif } static inline bool io_is_compat(struct io_ring_ctx *ctx) { return IS_ENABLED(CONFIG_COMPAT) && unlikely(ctx->compat); } static inline void io_req_task_work_add(struct io_kiocb *req) { __io_req_task_work_add(req, 0); } static inline void io_submit_flush_completions(struct io_ring_ctx *ctx) { if (!wq_list_empty(&ctx->submit_state.compl_reqs) || ctx->submit_state.cq_flush) __io_submit_flush_completions(ctx); } #define io_for_each_link(pos, head) \ for (pos = (head); pos; pos = pos->link) static inline bool io_get_cqe_overflow(struct io_ring_ctx *ctx, struct io_uring_cqe **ret, bool overflow, bool cqe32) { io_lockdep_assert_cq_locked(ctx); if (unlikely(ctx->cqe_sentinel - ctx->cqe_cached < (cqe32 + 1))) { if (unlikely(!io_cqe_cache_refill(ctx, overflow, cqe32))) return false; } *ret = ctx->cqe_cached; ctx->cached_cq_tail++; ctx->cqe_cached++; if (ctx->flags & IORING_SETUP_CQE32) { ctx->cqe_cached++; } else if (cqe32 && ctx->flags & IORING_SETUP_CQE_MIXED) { ctx->cqe_cached++; ctx->cached_cq_tail++; } WARN_ON_ONCE(ctx->cqe_cached > ctx->cqe_sentinel); return true; } static inline bool io_get_cqe(struct io_ring_ctx *ctx, struct io_uring_cqe **ret, bool cqe32) { return io_get_cqe_overflow(ctx, ret, false, cqe32); } static inline bool io_defer_get_uncommited_cqe(struct io_ring_ctx *ctx, struct io_uring_cqe **cqe_ret) { io_lockdep_assert_cq_locked(ctx); ctx->submit_state.cq_flush = true; return io_get_cqe(ctx, cqe_ret, ctx->flags & IORING_SETUP_CQE_MIXED); } static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx, struct io_kiocb *req) { bool is_cqe32 = req->cqe.flags & IORING_CQE_F_32; struct io_uring_cqe *cqe; /* * If we can't get a cq entry, userspace overflowed the submission * (by quite a lot). */ if (unlikely(!io_get_cqe(ctx, &cqe, is_cqe32))) return false; memcpy(cqe, &req->cqe, sizeof(*cqe)); if (ctx->flags & IORING_SETUP_CQE32 || is_cqe32) { memcpy(cqe->big_cqe, &req->big_cqe, sizeof(*cqe)); memset(&req->big_cqe, 0, sizeof(req->big_cqe)); } if (trace_io_uring_complete_enabled()) trace_io_uring_complete(req->ctx, req, cqe); return true; } static inline void req_set_fail(struct io_kiocb *req) { req->flags |= REQ_F_FAIL; if (req->flags & REQ_F_CQE_SKIP) { req->flags &= ~REQ_F_CQE_SKIP; req->flags |= REQ_F_SKIP_LINK_CQES; } } static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags) { req->cqe.res = res; req->cqe.flags = cflags; } static inline u32 ctx_cqe32_flags(struct io_ring_ctx *ctx) { if (ctx->flags & IORING_SETUP_CQE_MIXED) return IORING_CQE_F_32; return 0; } static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags, __u64 extra1, __u64 extra2) { req->cqe.res = res; req->cqe.flags = cflags | ctx_cqe32_flags(req->ctx); req->big_cqe.extra1 = extra1; req->big_cqe.extra2 = extra2; } static inline void *io_uring_alloc_async_data(struct io_alloc_cache *cache, struct io_kiocb *req) { if (cache) { req->async_data = io_cache_alloc(cache, GFP_KERNEL); } else { const struct io_issue_def *def = &io_issue_defs[req->opcode]; WARN_ON_ONCE(!def->async_size); req->async_data = kmalloc(def->async_size, GFP_KERNEL); } if (req->async_data) req->flags |= REQ_F_ASYNC_DATA; return req->async_data; } static inline bool req_has_async_data(struct io_kiocb *req) { return req->flags & REQ_F_ASYNC_DATA; } static inline void io_req_async_data_clear(struct io_kiocb *req, io_req_flags_t extra_flags) { req->flags &= ~(REQ_F_ASYNC_DATA|extra_flags); req->async_data = NULL; } static inline void io_req_async_data_free(struct io_kiocb *req) { kfree(req->async_data); io_req_async_data_clear(req, 0); } static inline void io_put_file(struct io_kiocb *req) { if (!(req->flags & REQ_F_FIXED_FILE) && req->file) fput(req->file); } static inline void io_ring_submit_unlock(struct io_ring_ctx *ctx, unsigned issue_flags) { lockdep_assert_held(&ctx->uring_lock); if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) mutex_unlock(&ctx->uring_lock); } static inline void io_ring_submit_lock(struct io_ring_ctx *ctx, unsigned issue_flags) { /* * "Normal" inline submissions always hold the uring_lock, since we * grab it from the system call. Same is true for the SQPOLL offload. * The only exception is when we've detached the request and issue it * from an async worker thread, grab the lock for that case. */ if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) mutex_lock(&ctx->uring_lock); lockdep_assert_held(&ctx->uring_lock); } static inline void io_commit_cqring(struct io_ring_ctx *ctx) { /* order cqe stores with ring update */ smp_store_release(&ctx->rings->cq.tail, ctx->cached_cq_tail); } static inline void __io_wq_wake(struct wait_queue_head *wq) { /* * * Pass in EPOLLIN|EPOLL_URING_WAKE as the poll wakeup key. The latter * set in the mask so that if we recurse back into our own poll * waitqueue handlers, we know we have a dependency between eventfd or * epoll and should terminate multishot poll at that point. */ if (wq_has_sleeper(wq)) __wake_up(wq, TASK_NORMAL, 0, poll_to_key(EPOLL_URING_WAKE | EPOLLIN)); } static inline void io_poll_wq_wake(struct io_ring_ctx *ctx) { __io_wq_wake(&ctx->poll_wq); } static inline void io_cqring_wake(struct io_ring_ctx *ctx) { /* * Trigger waitqueue handler on all waiters on our waitqueue. This * won't necessarily wake up all the tasks, io_should_wake() will make * that decision. */ __io_wq_wake(&ctx->cq_wait); } static inline bool io_sqring_full(struct io_ring_ctx *ctx) { struct io_rings *r = ctx->rings; /* * SQPOLL must use the actual sqring head, as using the cached_sq_head * is race prone if the SQPOLL thread has grabbed entries but not yet * committed them to the ring. For !SQPOLL, this doesn't matter, but * since this helper is just used for SQPOLL sqring waits (or POLLOUT), * just read the actual sqring head unconditionally. */ return READ_ONCE(r->sq.tail) - READ_ONCE(r->sq.head) == ctx->sq_entries; } static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx) { struct io_rings *rings = ctx->rings; unsigned int entries; /* make sure SQ entry isn't read before tail */ entries = smp_load_acquire(&rings->sq.tail) - ctx->cached_sq_head; return min(entries, ctx->sq_entries); } static inline int io_run_task_work(void) { bool ret = false; /* * Always check-and-clear the task_work notification signal. With how * signaling works for task_work, we can find it set with nothing to * run. We need to clear it for that case, like get_signal() does. */ if (test_thread_flag(TIF_NOTIFY_SIGNAL)) clear_notify_signal(); /* * PF_IO_WORKER never returns to userspace, so check here if we have * notify work that needs processing. */ if (current->flags & PF_IO_WORKER) { if (test_thread_flag(TIF_NOTIFY_RESUME)) { __set_current_state(TASK_RUNNING); resume_user_mode_work(NULL); } if (current->io_uring) { unsigned int count = 0; __set_current_state(TASK_RUNNING); tctx_task_work_run(current->io_uring, UINT_MAX, &count); if (count) ret = true; } } if (task_work_pending(current)) { __set_current_state(TASK_RUNNING); task_work_run(); ret = true; } return ret; } static inline bool io_local_work_pending(struct io_ring_ctx *ctx) { return !llist_empty(&ctx->work_llist) || !llist_empty(&ctx->retry_llist); } static inline bool io_task_work_pending(struct io_ring_ctx *ctx) { return task_work_pending(current) || io_local_work_pending(ctx); } static inline void io_tw_lock(struct io_ring_ctx *ctx, io_tw_token_t tw) { lockdep_assert_held(&ctx->uring_lock); } /* * Don't complete immediately but use deferred completion infrastructure. * Protected by ->uring_lock and can only be used either with * IO_URING_F_COMPLETE_DEFER or inside a tw handler holding the mutex. */ static inline void io_req_complete_defer(struct io_kiocb *req) __must_hold(&req->ctx->uring_lock) { struct io_submit_state *state = &req->ctx->submit_state; lockdep_assert_held(&req->ctx->uring_lock); wq_list_add_tail(&req->comp_list, &state->compl_reqs); } static inline void io_commit_cqring_flush(struct io_ring_ctx *ctx) { if (unlikely(ctx->off_timeout_used || ctx->has_evfd || ctx->poll_activated)) __io_commit_cqring_flush(ctx); } static inline void io_get_task_refs(int nr) { struct io_uring_task *tctx = current->io_uring; tctx->cached_refs -= nr; if (unlikely(tctx->cached_refs < 0)) io_task_refs_refill(tctx); } static inline bool io_req_cache_empty(struct io_ring_ctx *ctx) { return !ctx->submit_state.free_list.next; } extern struct kmem_cache *req_cachep; static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx) { struct io_kiocb *req; req = container_of(ctx->submit_state.free_list.next, struct io_kiocb, comp_list); wq_stack_extract(&ctx->submit_state.free_list); return req; } static inline bool io_alloc_req(struct io_ring_ctx *ctx, struct io_kiocb **req) { if (unlikely(io_req_cache_empty(ctx))) { if (!__io_alloc_req_refill(ctx)) return false; } *req = io_extract_req(ctx); return true; } static inline bool io_allowed_defer_tw_run(struct io_ring_ctx *ctx) { return likely(ctx->submitter_task == current); } static inline bool io_allowed_run_tw(struct io_ring_ctx *ctx) { return likely(!(ctx->flags & IORING_SETUP_DEFER_TASKRUN) || ctx->submitter_task == current); } /* * Terminate the request if either of these conditions are true: * * 1) It's being executed by the original task, but that task is marked * with PF_EXITING as it's exiting. * 2) PF_KTHREAD is set, in which case the invoker of the task_work is * our fallback task_work. */ static inline bool io_should_terminate_tw(struct io_ring_ctx *ctx) { return (current->flags & (PF_KTHREAD | PF_EXITING)) || percpu_ref_is_dying(&ctx->refs); } static inline void io_req_queue_tw_complete(struct io_kiocb *req, s32 res) { io_req_set_res(req, res, 0); req->io_task_work.func = io_req_task_complete; io_req_task_work_add(req); } /* * IORING_SETUP_SQE128 contexts allocate twice the normal SQE size for each * slot. */ static inline size_t uring_sqe_size(struct io_ring_ctx *ctx) { if (ctx->flags & IORING_SETUP_SQE128) return 2 * sizeof(struct io_uring_sqe); return sizeof(struct io_uring_sqe); } static inline bool io_file_can_poll(struct io_kiocb *req) { if (req->flags & REQ_F_CAN_POLL) return true; if (req->file && file_can_poll(req->file)) { req->flags |= REQ_F_CAN_POLL; return true; } return false; } static inline ktime_t io_get_time(struct io_ring_ctx *ctx) { if (ctx->clockid == CLOCK_MONOTONIC) return ktime_get(); return ktime_get_with_offset(ctx->clock_offset); } enum { IO_CHECK_CQ_OVERFLOW_BIT, IO_CHECK_CQ_DROPPED_BIT, }; static inline bool io_has_work(struct io_ring_ctx *ctx) { return test_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq) || io_local_work_pending(ctx); } #endif |
| 625 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | /* SPDX-License-Identifier: GPL-2.0 */ #undef TRACE_SYSTEM #define TRACE_SYSTEM bpf_test_run #if !defined(_TRACE_BPF_TEST_RUN_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_BPF_TEST_RUN_H #include <linux/tracepoint.h> TRACE_EVENT(bpf_trigger_tp, TP_PROTO(int nonce), TP_ARGS(nonce), TP_STRUCT__entry( __field(int, nonce) ), TP_fast_assign( __entry->nonce = nonce; ), TP_printk("nonce %d", __entry->nonce) ); DECLARE_EVENT_CLASS(bpf_test_finish, TP_PROTO(int *err), TP_ARGS(err), TP_STRUCT__entry( __field(int, err) ), TP_fast_assign( __entry->err = *err; ), TP_printk("bpf_test_finish with err=%d", __entry->err) ); #ifdef DEFINE_EVENT_WRITABLE #undef BPF_TEST_RUN_DEFINE_EVENT #define BPF_TEST_RUN_DEFINE_EVENT(template, call, proto, args, size) \ DEFINE_EVENT_WRITABLE(template, call, PARAMS(proto), \ PARAMS(args), size) #else #undef BPF_TEST_RUN_DEFINE_EVENT #define BPF_TEST_RUN_DEFINE_EVENT(template, call, proto, args, size) \ DEFINE_EVENT(template, call, PARAMS(proto), PARAMS(args)) #endif BPF_TEST_RUN_DEFINE_EVENT(bpf_test_finish, bpf_test_finish, TP_PROTO(int *err), TP_ARGS(err), sizeof(int) ); #endif /* This part must be outside protection */ #include <trace/define_trace.h> |
| 10 4 1 1 55 43 1 6 6 4 1 4 3 1 8 5 5 5 5 8 8 5 5 5 5 5 8 8 8 8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Hardware dependent layer * Copyright (c) by Jaroslav Kysela <perex@perex.cz> */ #include <linux/major.h> #include <linux/init.h> #include <linux/slab.h> #include <linux/time.h> #include <linux/mutex.h> #include <linux/module.h> #include <linux/sched/signal.h> #include <sound/core.h> #include <sound/control.h> #include <sound/minors.h> #include <sound/hwdep.h> #include <sound/info.h> MODULE_AUTHOR("Jaroslav Kysela <perex@perex.cz>"); MODULE_DESCRIPTION("Hardware dependent layer"); MODULE_LICENSE("GPL"); static LIST_HEAD(snd_hwdep_devices); static DEFINE_MUTEX(register_mutex); static int snd_hwdep_dev_free(struct snd_device *device); static int snd_hwdep_dev_register(struct snd_device *device); static int snd_hwdep_dev_disconnect(struct snd_device *device); static struct snd_hwdep *snd_hwdep_search(struct snd_card *card, int device) { struct snd_hwdep *hwdep; list_for_each_entry(hwdep, &snd_hwdep_devices, list) if (hwdep->card == card && hwdep->device == device) return hwdep; return NULL; } static loff_t snd_hwdep_llseek(struct file * file, loff_t offset, int orig) { struct snd_hwdep *hw = file->private_data; if (hw->ops.llseek) return hw->ops.llseek(hw, file, offset, orig); return -ENXIO; } static ssize_t snd_hwdep_read(struct file * file, char __user *buf, size_t count, loff_t *offset) { struct snd_hwdep *hw = file->private_data; if (hw->ops.read) return hw->ops.read(hw, buf, count, offset); return -ENXIO; } static ssize_t snd_hwdep_write(struct file * file, const char __user *buf, size_t count, loff_t *offset) { struct snd_hwdep *hw = file->private_data; if (hw->ops.write) return hw->ops.write(hw, buf, count, offset); return -ENXIO; } static int snd_hwdep_open(struct inode *inode, struct file * file) { int major = imajor(inode); struct snd_hwdep *hw; int err; wait_queue_entry_t wait; if (major == snd_major) { hw = snd_lookup_minor_data(iminor(inode), SNDRV_DEVICE_TYPE_HWDEP); #ifdef CONFIG_SND_OSSEMUL } else if (major == SOUND_MAJOR) { hw = snd_lookup_oss_minor_data(iminor(inode), SNDRV_OSS_DEVICE_TYPE_DMFM); #endif } else return -ENXIO; if (hw == NULL) return -ENODEV; if (!try_module_get(hw->card->module)) { snd_card_unref(hw->card); return -EFAULT; } init_waitqueue_entry(&wait, current); add_wait_queue(&hw->open_wait, &wait); mutex_lock(&hw->open_mutex); while (1) { if (hw->exclusive && hw->used > 0) { err = -EBUSY; break; } if (!hw->ops.open) { err = 0; break; } err = hw->ops.open(hw, file); if (err >= 0) break; if (err == -EAGAIN) { if (file->f_flags & O_NONBLOCK) { err = -EBUSY; break; } } else break; set_current_state(TASK_INTERRUPTIBLE); mutex_unlock(&hw->open_mutex); schedule(); mutex_lock(&hw->open_mutex); if (hw->card->shutdown) { err = -ENODEV; break; } if (signal_pending(current)) { err = -ERESTARTSYS; break; } } remove_wait_queue(&hw->open_wait, &wait); if (err >= 0) { err = snd_card_file_add(hw->card, file); if (err >= 0) { file->private_data = hw; hw->used++; } else { if (hw->ops.release) hw->ops.release(hw, file); } } mutex_unlock(&hw->open_mutex); if (err < 0) module_put(hw->card->module); snd_card_unref(hw->card); return err; } static int snd_hwdep_release(struct inode *inode, struct file * file) { int err = 0; struct snd_hwdep *hw = file->private_data; struct module *mod = hw->card->module; scoped_guard(mutex, &hw->open_mutex) { if (hw->ops.release) err = hw->ops.release(hw, file); if (hw->used > 0) hw->used--; } wake_up(&hw->open_wait); snd_card_file_remove(hw->card, file); module_put(mod); return err; } static __poll_t snd_hwdep_poll(struct file * file, poll_table * wait) { struct snd_hwdep *hw = file->private_data; if (hw->ops.poll) return hw->ops.poll(hw, file, wait); return 0; } static int snd_hwdep_info(struct snd_hwdep *hw, struct snd_hwdep_info __user *_info) { struct snd_hwdep_info info; memset(&info, 0, sizeof(info)); info.card = hw->card->number; strscpy(info.id, hw->id, sizeof(info.id)); strscpy(info.name, hw->name, sizeof(info.name)); info.iface = hw->iface; if (copy_to_user(_info, &info, sizeof(info))) return -EFAULT; return 0; } static int snd_hwdep_dsp_status(struct snd_hwdep *hw, struct snd_hwdep_dsp_status __user *_info) { struct snd_hwdep_dsp_status info; int err; if (! hw->ops.dsp_status) return -ENXIO; memset(&info, 0, sizeof(info)); info.dsp_loaded = hw->dsp_loaded; err = hw->ops.dsp_status(hw, &info); if (err < 0) return err; if (copy_to_user(_info, &info, sizeof(info))) return -EFAULT; return 0; } static int snd_hwdep_dsp_load(struct snd_hwdep *hw, struct snd_hwdep_dsp_image *info) { int err; if (! hw->ops.dsp_load) return -ENXIO; if (info->index >= 32) return -EINVAL; /* check whether the dsp was already loaded */ if (hw->dsp_loaded & (1u << info->index)) return -EBUSY; err = hw->ops.dsp_load(hw, info); if (err < 0) return err; hw->dsp_loaded |= (1u << info->index); return 0; } static int snd_hwdep_dsp_load_user(struct snd_hwdep *hw, struct snd_hwdep_dsp_image __user *_info) { struct snd_hwdep_dsp_image info = {}; if (copy_from_user(&info, _info, sizeof(info))) return -EFAULT; return snd_hwdep_dsp_load(hw, &info); } static long snd_hwdep_ioctl(struct file * file, unsigned int cmd, unsigned long arg) { struct snd_hwdep *hw = file->private_data; void __user *argp = (void __user *)arg; switch (cmd) { case SNDRV_HWDEP_IOCTL_PVERSION: return put_user(SNDRV_HWDEP_VERSION, (int __user *)argp); case SNDRV_HWDEP_IOCTL_INFO: return snd_hwdep_info(hw, argp); case SNDRV_HWDEP_IOCTL_DSP_STATUS: return snd_hwdep_dsp_status(hw, argp); case SNDRV_HWDEP_IOCTL_DSP_LOAD: return snd_hwdep_dsp_load_user(hw, argp); } if (hw->ops.ioctl) return hw->ops.ioctl(hw, file, cmd, arg); return -ENOTTY; } static int snd_hwdep_mmap(struct file * file, struct vm_area_struct * vma) { struct snd_hwdep *hw = file->private_data; if (hw->ops.mmap) return hw->ops.mmap(hw, file, vma); return -ENXIO; } static int snd_hwdep_control_ioctl(struct snd_card *card, struct snd_ctl_file * control, unsigned int cmd, unsigned long arg) { switch (cmd) { case SNDRV_CTL_IOCTL_HWDEP_NEXT_DEVICE: { int device; if (get_user(device, (int __user *)arg)) return -EFAULT; scoped_guard(mutex, ®ister_mutex) { if (device < 0) device = 0; else if (device < SNDRV_MINOR_HWDEPS) device++; else device = SNDRV_MINOR_HWDEPS; while (device < SNDRV_MINOR_HWDEPS) { if (snd_hwdep_search(card, device)) break; device++; } if (device >= SNDRV_MINOR_HWDEPS) device = -1; } if (put_user(device, (int __user *)arg)) return -EFAULT; return 0; } case SNDRV_CTL_IOCTL_HWDEP_INFO: { struct snd_hwdep_info __user *info = (struct snd_hwdep_info __user *)arg; int device; struct snd_hwdep *hwdep; if (get_user(device, &info->device)) return -EFAULT; scoped_guard(mutex, ®ister_mutex) { hwdep = snd_hwdep_search(card, device); if (!hwdep) return -ENXIO; return snd_hwdep_info(hwdep, info); } break; } } return -ENOIOCTLCMD; } #ifdef CONFIG_COMPAT #include "hwdep_compat.c" #else #define snd_hwdep_ioctl_compat NULL #endif /* */ static const struct file_operations snd_hwdep_f_ops = { .owner = THIS_MODULE, .llseek = snd_hwdep_llseek, .read = snd_hwdep_read, .write = snd_hwdep_write, .open = snd_hwdep_open, .release = snd_hwdep_release, .poll = snd_hwdep_poll, .unlocked_ioctl = snd_hwdep_ioctl, .compat_ioctl = snd_hwdep_ioctl_compat, .mmap = snd_hwdep_mmap, }; static void snd_hwdep_free(struct snd_hwdep *hwdep) { if (!hwdep) return; if (hwdep->private_free) hwdep->private_free(hwdep); put_device(hwdep->dev); kfree(hwdep); } /** * snd_hwdep_new - create a new hwdep instance * @card: the card instance * @id: the id string * @device: the device index (zero-based) * @rhwdep: the pointer to store the new hwdep instance * * Creates a new hwdep instance with the given index on the card. * The callbacks (hwdep->ops) must be set on the returned instance * after this call manually by the caller. * * Return: Zero if successful, or a negative error code on failure. */ int snd_hwdep_new(struct snd_card *card, char *id, int device, struct snd_hwdep **rhwdep) { struct snd_hwdep *hwdep; int err; static const struct snd_device_ops ops = { .dev_free = snd_hwdep_dev_free, .dev_register = snd_hwdep_dev_register, .dev_disconnect = snd_hwdep_dev_disconnect, }; if (snd_BUG_ON(!card)) return -ENXIO; if (rhwdep) *rhwdep = NULL; hwdep = kzalloc(sizeof(*hwdep), GFP_KERNEL); if (!hwdep) return -ENOMEM; init_waitqueue_head(&hwdep->open_wait); mutex_init(&hwdep->open_mutex); hwdep->card = card; hwdep->device = device; if (id) strscpy(hwdep->id, id, sizeof(hwdep->id)); err = snd_device_alloc(&hwdep->dev, card); if (err < 0) { snd_hwdep_free(hwdep); return err; } dev_set_name(hwdep->dev, "hwC%iD%i", card->number, device); #ifdef CONFIG_SND_OSSEMUL hwdep->oss_type = -1; #endif err = snd_device_new(card, SNDRV_DEV_HWDEP, hwdep, &ops); if (err < 0) { snd_hwdep_free(hwdep); return err; } if (rhwdep) *rhwdep = hwdep; return 0; } EXPORT_SYMBOL(snd_hwdep_new); static int snd_hwdep_dev_free(struct snd_device *device) { snd_hwdep_free(device->device_data); return 0; } static int snd_hwdep_dev_register(struct snd_device *device) { struct snd_hwdep *hwdep = device->device_data; struct snd_card *card = hwdep->card; int err; guard(mutex)(®ister_mutex); if (snd_hwdep_search(card, hwdep->device)) return -EBUSY; list_add_tail(&hwdep->list, &snd_hwdep_devices); err = snd_register_device(SNDRV_DEVICE_TYPE_HWDEP, hwdep->card, hwdep->device, &snd_hwdep_f_ops, hwdep, hwdep->dev); if (err < 0) { dev_err(hwdep->dev, "unable to register\n"); list_del(&hwdep->list); return err; } #ifdef CONFIG_SND_OSSEMUL hwdep->ossreg = 0; if (hwdep->oss_type >= 0) { if (hwdep->oss_type == SNDRV_OSS_DEVICE_TYPE_DMFM && hwdep->device) dev_warn(hwdep->dev, "only hwdep device 0 can be registered as OSS direct FM device!\n"); else if (snd_register_oss_device(hwdep->oss_type, card, hwdep->device, &snd_hwdep_f_ops, hwdep) < 0) dev_warn(hwdep->dev, "unable to register OSS compatibility device\n"); else hwdep->ossreg = 1; } #endif return 0; } static int snd_hwdep_dev_disconnect(struct snd_device *device) { struct snd_hwdep *hwdep = device->device_data; if (snd_BUG_ON(!hwdep)) return -ENXIO; guard(mutex)(®ister_mutex); if (snd_hwdep_search(hwdep->card, hwdep->device) != hwdep) return -EINVAL; guard(mutex)(&hwdep->open_mutex); wake_up(&hwdep->open_wait); #ifdef CONFIG_SND_OSSEMUL if (hwdep->ossreg) snd_unregister_oss_device(hwdep->oss_type, hwdep->card, hwdep->device); #endif snd_unregister_device(hwdep->dev); list_del_init(&hwdep->list); return 0; } #ifdef CONFIG_SND_PROC_FS /* * Info interface */ static void snd_hwdep_proc_read(struct snd_info_entry *entry, struct snd_info_buffer *buffer) { struct snd_hwdep *hwdep; guard(mutex)(®ister_mutex); list_for_each_entry(hwdep, &snd_hwdep_devices, list) snd_iprintf(buffer, "%02i-%02i: %s\n", hwdep->card->number, hwdep->device, hwdep->name); } static struct snd_info_entry *snd_hwdep_proc_entry; static void __init snd_hwdep_proc_init(void) { struct snd_info_entry *entry; entry = snd_info_create_module_entry(THIS_MODULE, "hwdep", NULL); if (entry) { entry->c.text.read = snd_hwdep_proc_read; if (snd_info_register(entry) < 0) { snd_info_free_entry(entry); entry = NULL; } } snd_hwdep_proc_entry = entry; } static void __exit snd_hwdep_proc_done(void) { snd_info_free_entry(snd_hwdep_proc_entry); } #else /* !CONFIG_SND_PROC_FS */ #define snd_hwdep_proc_init() #define snd_hwdep_proc_done() #endif /* CONFIG_SND_PROC_FS */ /* * ENTRY functions */ static int __init alsa_hwdep_init(void) { snd_hwdep_proc_init(); snd_ctl_register_ioctl(snd_hwdep_control_ioctl); snd_ctl_register_ioctl_compat(snd_hwdep_control_ioctl); return 0; } static void __exit alsa_hwdep_exit(void) { snd_ctl_unregister_ioctl(snd_hwdep_control_ioctl); snd_ctl_unregister_ioctl_compat(snd_hwdep_control_ioctl); snd_hwdep_proc_done(); } module_init(alsa_hwdep_init) module_exit(alsa_hwdep_exit) |
| 36 36 36 36 36 36 30 30 4 32 31 32 31 32 32 31 31 36 36 15 12 12 454 455 36 36 36 36 36 35 36 36 36 36 36 8 3 5 434 139 140 139 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 | // SPDX-License-Identifier: GPL-2.0-or-later #include <net/psp.h> #include <net/gro.h> #include <net/dst_metadata.h> #include <net/busy_poll.h> #include <trace/events/net.h> #include <linux/skbuff_ref.h> #define MAX_GRO_SKBS 8 static DEFINE_SPINLOCK(offload_lock); /** * dev_add_offload - register offload handlers * @po: protocol offload declaration * * Add protocol offload handlers to the networking stack. The passed * &proto_offload is linked into kernel lists and may not be freed until * it has been removed from the kernel lists. * * This call does not sleep therefore it can not * guarantee all CPU's that are in middle of receiving packets * will see the new offload handlers (until the next received packet). */ void dev_add_offload(struct packet_offload *po) { struct packet_offload *elem; spin_lock(&offload_lock); list_for_each_entry(elem, &net_hotdata.offload_base, list) { if (po->priority < elem->priority) break; } list_add_rcu(&po->list, elem->list.prev); spin_unlock(&offload_lock); } EXPORT_SYMBOL(dev_add_offload); /** * __dev_remove_offload - remove offload handler * @po: packet offload declaration * * Remove a protocol offload handler that was previously added to the * kernel offload handlers by dev_add_offload(). The passed &offload_type * is removed from the kernel lists and can be freed or reused once this * function returns. * * The packet type might still be in use by receivers * and must not be freed until after all the CPU's have gone * through a quiescent state. */ static void __dev_remove_offload(struct packet_offload *po) { struct list_head *head = &net_hotdata.offload_base; struct packet_offload *po1; spin_lock(&offload_lock); list_for_each_entry(po1, head, list) { if (po == po1) { list_del_rcu(&po->list); goto out; } } pr_warn("dev_remove_offload: %p not found\n", po); out: spin_unlock(&offload_lock); } /** * dev_remove_offload - remove packet offload handler * @po: packet offload declaration * * Remove a packet offload handler that was previously added to the kernel * offload handlers by dev_add_offload(). The passed &offload_type is * removed from the kernel lists and can be freed or reused once this * function returns. * * This call sleeps to guarantee that no CPU is looking at the packet * type after return. */ void dev_remove_offload(struct packet_offload *po) { __dev_remove_offload(po); synchronize_net(); } EXPORT_SYMBOL(dev_remove_offload); int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) { struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); unsigned int offset = skb_gro_offset(skb); unsigned int headlen = skb_headlen(skb); unsigned int len = skb_gro_len(skb); unsigned int delta_truesize; unsigned int new_truesize; struct sk_buff *lp; int segs; /* Do not splice page pool based packets w/ non-page pool * packets. This can result in reference count issues as page * pool pages will not decrement the reference count and will * instead be immediately returned to the pool or have frag * count decremented. */ if (p->pp_recycle != skb->pp_recycle) return -ETOOMANYREFS; if (unlikely(p->len + len >= netif_get_gro_max_size(p->dev, p) || NAPI_GRO_CB(skb)->flush)) return -E2BIG; if (unlikely(p->len + len >= GRO_LEGACY_MAX_SIZE)) { if (NAPI_GRO_CB(skb)->proto != IPPROTO_TCP || (p->protocol == htons(ETH_P_IPV6) && skb_headroom(p) < sizeof(struct hop_jumbo_hdr)) || p->encapsulation) return -E2BIG; } segs = NAPI_GRO_CB(skb)->count; lp = NAPI_GRO_CB(p)->last; pinfo = skb_shinfo(lp); if (headlen <= offset) { skb_frag_t *frag; skb_frag_t *frag2; int i = skbinfo->nr_frags; int nr_frags = pinfo->nr_frags + i; if (nr_frags > MAX_SKB_FRAGS) goto merge; offset -= headlen; pinfo->nr_frags = nr_frags; skbinfo->nr_frags = 0; frag = pinfo->frags + nr_frags; frag2 = skbinfo->frags + i; do { *--frag = *--frag2; } while (--i); skb_frag_off_add(frag, offset); skb_frag_size_sub(frag, offset); /* all fragments truesize : remove (head size + sk_buff) */ new_truesize = SKB_TRUESIZE(skb_end_offset(skb)); delta_truesize = skb->truesize - new_truesize; skb->truesize = new_truesize; skb->len -= skb->data_len; skb->data_len = 0; NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; goto done; } else if (skb->head_frag) { int nr_frags = pinfo->nr_frags; skb_frag_t *frag = pinfo->frags + nr_frags; struct page *page = virt_to_head_page(skb->head); unsigned int first_size = headlen - offset; unsigned int first_offset; if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) goto merge; first_offset = skb->data - (unsigned char *)page_address(page) + offset; pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; skb_frag_fill_page_desc(frag, page, first_offset, first_size); memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); /* We dont need to clear skbinfo->nr_frags here */ new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff)); delta_truesize = skb->truesize - new_truesize; skb->truesize = new_truesize; NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; goto done; } merge: /* sk ownership - if any - completely transferred to the aggregated packet */ skb->destructor = NULL; skb->sk = NULL; delta_truesize = skb->truesize; if (offset > headlen) { unsigned int eat = offset - headlen; skb_frag_off_add(&skbinfo->frags[0], eat); skb_frag_size_sub(&skbinfo->frags[0], eat); skb->data_len -= eat; skb->len -= eat; offset = headlen; } __skb_pull(skb, offset); if (NAPI_GRO_CB(p)->last == p) skb_shinfo(p)->frag_list = skb; else NAPI_GRO_CB(p)->last->next = skb; NAPI_GRO_CB(p)->last = skb; __skb_header_release(skb); lp = p; done: NAPI_GRO_CB(p)->count += segs; p->data_len += len; p->truesize += delta_truesize; p->len += len; if (lp != p) { lp->data_len += len; lp->truesize += delta_truesize; lp->len += len; } NAPI_GRO_CB(skb)->same_flow = 1; return 0; } int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) { if (unlikely(p->len + skb->len >= 65536)) return -E2BIG; if (NAPI_GRO_CB(p)->last == p) skb_shinfo(p)->frag_list = skb; else NAPI_GRO_CB(p)->last->next = skb; skb_pull(skb, skb_gro_offset(skb)); NAPI_GRO_CB(p)->last = skb; NAPI_GRO_CB(p)->count++; p->data_len += skb->len; /* sk ownership - if any - completely transferred to the aggregated packet */ skb->destructor = NULL; skb->sk = NULL; p->truesize += skb->truesize; p->len += skb->len; NAPI_GRO_CB(skb)->same_flow = 1; return 0; } static void gro_complete(struct gro_node *gro, struct sk_buff *skb) { struct list_head *head = &net_hotdata.offload_base; struct packet_offload *ptype; __be16 type = skb->protocol; int err = -ENOENT; BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); if (NAPI_GRO_CB(skb)->count == 1) { skb_shinfo(skb)->gso_size = 0; goto out; } rcu_read_lock(); list_for_each_entry_rcu(ptype, head, list) { if (ptype->type != type || !ptype->callbacks.gro_complete) continue; err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete, ipv6_gro_complete, inet_gro_complete, skb, 0); break; } rcu_read_unlock(); if (err) { WARN_ON(&ptype->list == head); kfree_skb(skb); return; } out: gro_normal_one(gro, skb, NAPI_GRO_CB(skb)->count); } static void __gro_flush_chain(struct gro_node *gro, u32 index, bool flush_old) { struct list_head *head = &gro->hash[index].list; struct sk_buff *skb, *p; list_for_each_entry_safe_reverse(skb, p, head, list) { if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) return; skb_list_del_init(skb); gro_complete(gro, skb); gro->hash[index].count--; } if (!gro->hash[index].count) __clear_bit(index, &gro->bitmask); } /* * gro->hash[].list contains packets ordered by age. * youngest packets at the head of it. * Complete skbs in reverse order to reduce latencies. */ void __gro_flush(struct gro_node *gro, bool flush_old) { unsigned long bitmask = gro->bitmask; unsigned int i, base = ~0U; while ((i = ffs(bitmask)) != 0) { bitmask >>= i; base += i; __gro_flush_chain(gro, base, flush_old); } } EXPORT_SYMBOL(__gro_flush); static unsigned long gro_list_prepare_tc_ext(const struct sk_buff *skb, const struct sk_buff *p, unsigned long diffs) { #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) struct tc_skb_ext *skb_ext; struct tc_skb_ext *p_ext; skb_ext = skb_ext_find(skb, TC_SKB_EXT); p_ext = skb_ext_find(p, TC_SKB_EXT); diffs |= (!!p_ext) ^ (!!skb_ext); if (!diffs && unlikely(skb_ext)) diffs |= p_ext->chain ^ skb_ext->chain; #endif return diffs; } static void gro_list_prepare(const struct list_head *head, const struct sk_buff *skb) { unsigned int maclen = skb->dev->hard_header_len; u32 hash = skb_get_hash_raw(skb); struct sk_buff *p; list_for_each_entry(p, head, list) { unsigned long diffs; if (hash != skb_get_hash_raw(p)) { NAPI_GRO_CB(p)->same_flow = 0; continue; } diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; diffs |= p->vlan_all ^ skb->vlan_all; diffs |= skb_metadata_differs(p, skb); if (maclen == ETH_HLEN) diffs |= compare_ether_header(skb_mac_header(p), skb_mac_header(skb)); else if (!diffs) diffs = memcmp(skb_mac_header(p), skb_mac_header(skb), maclen); /* in most common scenarios 'slow_gro' is 0 * otherwise we are already on some slower paths * either skip all the infrequent tests altogether or * avoid trying too hard to skip each of them individually */ if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) { diffs |= p->sk != skb->sk; diffs |= skb_metadata_dst_cmp(p, skb); diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb); diffs |= gro_list_prepare_tc_ext(skb, p, diffs); diffs |= __psp_skb_coalesce_diff(skb, p, diffs); } NAPI_GRO_CB(p)->same_flow = !diffs; } } static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff) { const struct skb_shared_info *pinfo; const skb_frag_t *frag0; unsigned int headlen; NAPI_GRO_CB(skb)->network_offset = 0; NAPI_GRO_CB(skb)->data_offset = 0; headlen = skb_headlen(skb); NAPI_GRO_CB(skb)->frag0 = skb->data; NAPI_GRO_CB(skb)->frag0_len = headlen; if (headlen) return; pinfo = skb_shinfo(skb); frag0 = &pinfo->frags[0]; if (pinfo->nr_frags && skb_frag_page(frag0) && !PageHighMem(skb_frag_page(frag0)) && (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) { NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, skb_frag_size(frag0), skb->end - skb->tail); } } static void gro_pull_from_frag0(struct sk_buff *skb, int grow) { struct skb_shared_info *pinfo = skb_shinfo(skb); BUG_ON(skb->end - skb->tail < grow); memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); skb->data_len -= grow; skb->tail += grow; skb_frag_off_add(&pinfo->frags[0], grow); skb_frag_size_sub(&pinfo->frags[0], grow); if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { skb_frag_unref(skb, 0); memmove(pinfo->frags, pinfo->frags + 1, --pinfo->nr_frags * sizeof(pinfo->frags[0])); } } static void gro_try_pull_from_frag0(struct sk_buff *skb) { int grow = skb_gro_offset(skb) - skb_headlen(skb); if (grow > 0) gro_pull_from_frag0(skb, grow); } static void gro_flush_oldest(struct gro_node *gro, struct list_head *head) { struct sk_buff *oldest; oldest = list_last_entry(head, struct sk_buff, list); /* We are called with head length >= MAX_GRO_SKBS, so this is * impossible. */ if (WARN_ON_ONCE(!oldest)) return; /* Do not adjust napi->gro_hash[].count, caller is adding a new * SKB to the chain. */ skb_list_del_init(oldest); gro_complete(gro, oldest); } static enum gro_result dev_gro_receive(struct gro_node *gro, struct sk_buff *skb) { u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1); struct list_head *head = &net_hotdata.offload_base; struct gro_list *gro_list = &gro->hash[bucket]; struct packet_offload *ptype; __be16 type = skb->protocol; struct sk_buff *pp = NULL; enum gro_result ret; int same_flow; if (netif_elide_gro(skb->dev)) goto normal; gro_list_prepare(&gro_list->list, skb); rcu_read_lock(); list_for_each_entry_rcu(ptype, head, list) { if (ptype->type == type && ptype->callbacks.gro_receive) goto found_ptype; } rcu_read_unlock(); goto normal; found_ptype: skb_set_network_header(skb, skb_gro_offset(skb)); skb_reset_mac_len(skb); BUILD_BUG_ON(sizeof_field(struct napi_gro_cb, zeroed) != sizeof(u32)); BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct napi_gro_cb, zeroed), sizeof(u32))); /* Avoid slow unaligned acc */ *(u32 *)&NAPI_GRO_CB(skb)->zeroed = 0; NAPI_GRO_CB(skb)->flush = skb_has_frag_list(skb); NAPI_GRO_CB(skb)->count = 1; if (unlikely(skb_is_gso(skb))) { NAPI_GRO_CB(skb)->count = skb_shinfo(skb)->gso_segs; /* Only support TCP and non DODGY users. */ if (!skb_is_gso_tcp(skb) || (skb_shinfo(skb)->gso_type & SKB_GSO_DODGY)) NAPI_GRO_CB(skb)->flush = 1; } /* Setup for GRO checksum validation */ switch (skb->ip_summed) { case CHECKSUM_COMPLETE: NAPI_GRO_CB(skb)->csum = skb->csum; NAPI_GRO_CB(skb)->csum_valid = 1; break; case CHECKSUM_UNNECESSARY: NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; break; } pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive, ipv6_gro_receive, inet_gro_receive, &gro_list->list, skb); rcu_read_unlock(); if (PTR_ERR(pp) == -EINPROGRESS) { ret = GRO_CONSUMED; goto ok; } same_flow = NAPI_GRO_CB(skb)->same_flow; ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; if (pp) { skb_list_del_init(pp); gro_complete(gro, pp); gro_list->count--; } if (same_flow) goto ok; if (NAPI_GRO_CB(skb)->flush) goto normal; if (unlikely(gro_list->count >= MAX_GRO_SKBS)) gro_flush_oldest(gro, &gro_list->list); else gro_list->count++; /* Must be called before setting NAPI_GRO_CB(skb)->{age|last} */ gro_try_pull_from_frag0(skb); NAPI_GRO_CB(skb)->age = jiffies; NAPI_GRO_CB(skb)->last = skb; if (!skb_is_gso(skb)) skb_shinfo(skb)->gso_size = skb_gro_len(skb); list_add(&skb->list, &gro_list->list); ret = GRO_HELD; ok: if (gro_list->count) { if (!test_bit(bucket, &gro->bitmask)) __set_bit(bucket, &gro->bitmask); } else if (test_bit(bucket, &gro->bitmask)) { __clear_bit(bucket, &gro->bitmask); } return ret; normal: ret = GRO_NORMAL; gro_try_pull_from_frag0(skb); goto ok; } struct packet_offload *gro_find_receive_by_type(__be16 type) { struct list_head *offload_head = &net_hotdata.offload_base; struct packet_offload *ptype; list_for_each_entry_rcu(ptype, offload_head, list) { if (ptype->type != type || !ptype->callbacks.gro_receive) continue; return ptype; } return NULL; } EXPORT_SYMBOL(gro_find_receive_by_type); struct packet_offload *gro_find_complete_by_type(__be16 type) { struct list_head *offload_head = &net_hotdata.offload_base; struct packet_offload *ptype; list_for_each_entry_rcu(ptype, offload_head, list) { if (ptype->type != type || !ptype->callbacks.gro_complete) continue; return ptype; } return NULL; } EXPORT_SYMBOL(gro_find_complete_by_type); static gro_result_t gro_skb_finish(struct gro_node *gro, struct sk_buff *skb, gro_result_t ret) { switch (ret) { case GRO_NORMAL: gro_normal_one(gro, skb, 1); break; case GRO_MERGED_FREE: if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) napi_skb_free_stolen_head(skb); else if (skb->fclone != SKB_FCLONE_UNAVAILABLE) __kfree_skb(skb); else __napi_kfree_skb(skb, SKB_CONSUMED); break; case GRO_HELD: case GRO_MERGED: case GRO_CONSUMED: break; } return ret; } gro_result_t gro_receive_skb(struct gro_node *gro, struct sk_buff *skb) { gro_result_t ret; __skb_mark_napi_id(skb, gro); trace_napi_gro_receive_entry(skb); skb_gro_reset_offset(skb, 0); ret = gro_skb_finish(gro, skb, dev_gro_receive(gro, skb)); trace_napi_gro_receive_exit(ret); return ret; } EXPORT_SYMBOL(gro_receive_skb); static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) { struct skb_shared_info *shinfo; if (unlikely(skb->pfmemalloc)) { consume_skb(skb); return; } __skb_pull(skb, skb_headlen(skb)); /* restore the reserve we had after netdev_alloc_skb_ip_align() */ skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); __vlan_hwaccel_clear_tag(skb); skb->dev = napi->dev; skb->skb_iif = 0; /* eth_type_trans() assumes pkt_type is PACKET_HOST */ skb->pkt_type = PACKET_HOST; skb->encapsulation = 0; skb->ip_summed = CHECKSUM_NONE; shinfo = skb_shinfo(skb); shinfo->gso_type = 0; shinfo->gso_size = 0; shinfo->hwtstamps.hwtstamp = 0; if (unlikely(skb->slow_gro)) { skb_orphan(skb); skb_ext_reset(skb); nf_reset_ct(skb); skb->slow_gro = 0; } napi->skb = skb; } struct sk_buff *napi_get_frags(struct napi_struct *napi) { struct sk_buff *skb = napi->skb; if (!skb) { skb = napi_alloc_skb(napi, GRO_MAX_HEAD); if (skb) { napi->skb = skb; skb_mark_napi_id(skb, napi); } } return skb; } EXPORT_SYMBOL(napi_get_frags); static gro_result_t napi_frags_finish(struct napi_struct *napi, struct sk_buff *skb, gro_result_t ret) { switch (ret) { case GRO_NORMAL: case GRO_HELD: __skb_push(skb, ETH_HLEN); skb->protocol = eth_type_trans(skb, skb->dev); if (ret == GRO_NORMAL) gro_normal_one(&napi->gro, skb, 1); break; case GRO_MERGED_FREE: if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) napi_skb_free_stolen_head(skb); else napi_reuse_skb(napi, skb); break; case GRO_MERGED: case GRO_CONSUMED: break; } return ret; } /* Upper GRO stack assumes network header starts at gro_offset=0 * Drivers could call both napi_gro_frags() and napi_gro_receive() * We copy ethernet header into skb->data to have a common layout. */ static struct sk_buff *napi_frags_skb(struct napi_struct *napi) { struct sk_buff *skb = napi->skb; const struct ethhdr *eth; unsigned int hlen = sizeof(*eth); napi->skb = NULL; skb_reset_mac_header(skb); skb_gro_reset_offset(skb, hlen); if (unlikely(!skb_gro_may_pull(skb, hlen))) { eth = skb_gro_header_slow(skb, hlen, 0); if (unlikely(!eth)) { net_warn_ratelimited("%s: dropping impossible skb from %s\n", __func__, napi->dev->name); napi_reuse_skb(napi, skb); return NULL; } } else { eth = (const struct ethhdr *)skb->data; if (NAPI_GRO_CB(skb)->frag0 != skb->data) gro_pull_from_frag0(skb, hlen); NAPI_GRO_CB(skb)->frag0 += hlen; NAPI_GRO_CB(skb)->frag0_len -= hlen; } __skb_pull(skb, hlen); /* * This works because the only protocols we care about don't require * special handling. * We'll fix it up properly in napi_frags_finish() */ skb->protocol = eth->h_proto; return skb; } gro_result_t napi_gro_frags(struct napi_struct *napi) { gro_result_t ret; struct sk_buff *skb = napi_frags_skb(napi); trace_napi_gro_frags_entry(skb); ret = napi_frags_finish(napi, skb, dev_gro_receive(&napi->gro, skb)); trace_napi_gro_frags_exit(ret); return ret; } EXPORT_SYMBOL(napi_gro_frags); /* Compute the checksum from gro_offset and return the folded value * after adding in any pseudo checksum. */ __sum16 __skb_gro_checksum_complete(struct sk_buff *skb) { __wsum wsum; __sum16 sum; wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); /* See comments in __skb_checksum_complete(). */ if (likely(!sum)) { if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && !skb->csum_complete_sw) netdev_rx_csum_fault(skb->dev, skb); } NAPI_GRO_CB(skb)->csum = wsum; NAPI_GRO_CB(skb)->csum_valid = 1; return sum; } EXPORT_SYMBOL(__skb_gro_checksum_complete); void gro_init(struct gro_node *gro) { for (u32 i = 0; i < GRO_HASH_BUCKETS; i++) { INIT_LIST_HEAD(&gro->hash[i].list); gro->hash[i].count = 0; } gro->bitmask = 0; gro->cached_napi_id = 0; INIT_LIST_HEAD(&gro->rx_list); gro->rx_count = 0; } void gro_cleanup(struct gro_node *gro) { struct sk_buff *skb, *n; for (u32 i = 0; i < GRO_HASH_BUCKETS; i++) { list_for_each_entry_safe(skb, n, &gro->hash[i].list, list) kfree_skb(skb); gro->hash[i].count = 0; } gro->bitmask = 0; gro->cached_napi_id = 0; list_for_each_entry_safe(skb, n, &gro->rx_list, list) kfree_skb(skb); gro->rx_count = 0; } |
| 155 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_X86_UNWIND_H #define _ASM_X86_UNWIND_H #include <linux/sched.h> #include <linux/ftrace.h> #include <linux/rethook.h> #include <asm/ptrace.h> #include <asm/stacktrace.h> #define IRET_FRAME_OFFSET (offsetof(struct pt_regs, ip)) #define IRET_FRAME_SIZE (sizeof(struct pt_regs) - IRET_FRAME_OFFSET) struct unwind_state { struct stack_info stack_info; unsigned long stack_mask; struct task_struct *task; int graph_idx; #if defined(CONFIG_RETHOOK) struct llist_node *kr_cur; #endif bool error; #if defined(CONFIG_UNWINDER_ORC) bool signal, full_regs; unsigned long sp, bp, ip; struct pt_regs *regs, *prev_regs; #elif defined(CONFIG_UNWINDER_FRAME_POINTER) bool got_irq; unsigned long *bp, *orig_sp, ip; /* * If non-NULL: The current frame is incomplete and doesn't contain a * valid BP. When looking for the next frame, use this instead of the * non-existent saved BP. */ unsigned long *next_bp; struct pt_regs *regs; #else unsigned long *sp; #endif }; void __unwind_start(struct unwind_state *state, struct task_struct *task, struct pt_regs *regs, unsigned long *first_frame); bool unwind_next_frame(struct unwind_state *state); unsigned long unwind_get_return_address(struct unwind_state *state); unsigned long *unwind_get_return_address_ptr(struct unwind_state *state); static inline bool unwind_done(struct unwind_state *state) { return state->stack_info.type == STACK_TYPE_UNKNOWN; } static inline bool unwind_error(struct unwind_state *state) { return state->error; } static inline void unwind_start(struct unwind_state *state, struct task_struct *task, struct pt_regs *regs, unsigned long *first_frame) { first_frame = first_frame ? : get_stack_pointer(task, regs); __unwind_start(state, task, regs, first_frame); } #if defined(CONFIG_UNWINDER_ORC) || defined(CONFIG_UNWINDER_FRAME_POINTER) /* * If 'partial' returns true, only the iret frame registers are valid. */ static inline struct pt_regs *unwind_get_entry_regs(struct unwind_state *state, bool *partial) { if (unwind_done(state)) return NULL; if (partial) { #ifdef CONFIG_UNWINDER_ORC *partial = !state->full_regs; #else *partial = false; #endif } return state->regs; } #else static inline struct pt_regs *unwind_get_entry_regs(struct unwind_state *state, bool *partial) { return NULL; } #endif #ifdef CONFIG_UNWINDER_ORC void unwind_init(void); void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size); #else static inline void unwind_init(void) {} static inline void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size) {} #endif static inline unsigned long unwind_recover_rethook(struct unwind_state *state, unsigned long addr, unsigned long *addr_p) { #ifdef CONFIG_RETHOOK if (is_rethook_trampoline(addr)) return rethook_find_ret_addr(state->task, (unsigned long)addr_p, &state->kr_cur); #endif return addr; } /* Recover the return address modified by rethook and ftrace_graph. */ static inline unsigned long unwind_recover_ret_addr(struct unwind_state *state, unsigned long addr, unsigned long *addr_p) { unsigned long ret; ret = ftrace_graph_ret_addr(state->task, &state->graph_idx, addr, addr_p); return unwind_recover_rethook(state, ret, addr_p); } /* * This disables KASAN checking when reading a value from another task's stack, * since the other task could be running on another CPU and could have poisoned * the stack in the meantime. */ #define READ_ONCE_TASK_STACK(task, x) \ ({ \ unsigned long val; \ if (task == current) \ val = READ_ONCE(x); \ else \ val = READ_ONCE_NOCHECK(x); \ val; \ }) static inline bool task_on_another_cpu(struct task_struct *task) { #ifdef CONFIG_SMP return task != current && task->on_cpu; #else return false; #endif } #endif /* _ASM_X86_UNWIND_H */ |
| 7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 | // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/completion.h> #include <linux/buffer_head.h> #include <linux/module.h> #include <linux/init.h> #include <linux/gfs2_ondisk.h> #include <linux/rcupdate.h> #include <linux/rculist_bl.h> #include <linux/atomic.h> #include <linux/mempool.h> #include "gfs2.h" #include "incore.h" #include "super.h" #include "sys.h" #include "util.h" #include "glock.h" #include "quota.h" #include "recovery.h" #include "dir.h" #include "glops.h" struct workqueue_struct *gfs2_control_wq; static void gfs2_init_inode_once(void *foo) { struct gfs2_inode *ip = foo; inode_init_once(&ip->i_inode); atomic_set(&ip->i_sizehint, 0); init_rwsem(&ip->i_rw_mutex); INIT_LIST_HEAD(&ip->i_ordered); ip->i_qadata = NULL; gfs2_holder_mark_uninitialized(&ip->i_rgd_gh); memset(&ip->i_res, 0, sizeof(ip->i_res)); RB_CLEAR_NODE(&ip->i_res.rs_node); ip->i_hash_cache = NULL; gfs2_holder_mark_uninitialized(&ip->i_iopen_gh); } static void gfs2_init_glock_once(void *foo) { struct gfs2_glock *gl = foo; INIT_LIST_HEAD(&gl->gl_holders); INIT_LIST_HEAD(&gl->gl_lru); INIT_LIST_HEAD(&gl->gl_ail_list); atomic_set(&gl->gl_ail_count, 0); atomic_set(&gl->gl_revokes, 0); } static void gfs2_init_gl_aspace_once(void *foo) { struct gfs2_glock_aspace *gla = foo; gfs2_init_glock_once(&gla->glock); address_space_init_once(&gla->mapping); } /** * init_gfs2_fs - Register GFS2 as a filesystem * * Returns: 0 on success, error code on failure */ static int __init init_gfs2_fs(void) { int error; gfs2_str2qstr(&gfs2_qdot, "."); gfs2_str2qstr(&gfs2_qdotdot, ".."); gfs2_quota_hash_init(); error = gfs2_sys_init(); if (error) return error; error = list_lru_init(&gfs2_qd_lru); if (error) goto fail_lru; error = gfs2_glock_init(); if (error) goto fail_glock; error = -ENOMEM; gfs2_glock_cachep = kmem_cache_create("gfs2_glock", sizeof(struct gfs2_glock), 0, SLAB_RECLAIM_ACCOUNT, gfs2_init_glock_once); if (!gfs2_glock_cachep) goto fail_cachep1; gfs2_glock_aspace_cachep = kmem_cache_create("gfs2_glock(aspace)", sizeof(struct gfs2_glock_aspace), 0, 0, gfs2_init_gl_aspace_once); if (!gfs2_glock_aspace_cachep) goto fail_cachep2; gfs2_inode_cachep = kmem_cache_create("gfs2_inode", sizeof(struct gfs2_inode), 0, SLAB_RECLAIM_ACCOUNT| SLAB_ACCOUNT, gfs2_init_inode_once); if (!gfs2_inode_cachep) goto fail_cachep3; gfs2_bufdata_cachep = kmem_cache_create("gfs2_bufdata", sizeof(struct gfs2_bufdata), 0, 0, NULL); if (!gfs2_bufdata_cachep) goto fail_cachep4; gfs2_rgrpd_cachep = kmem_cache_create("gfs2_rgrpd", sizeof(struct gfs2_rgrpd), 0, 0, NULL); if (!gfs2_rgrpd_cachep) goto fail_cachep5; gfs2_quotad_cachep = kmem_cache_create("gfs2_quotad", sizeof(struct gfs2_quota_data), 0, SLAB_RECLAIM_ACCOUNT, NULL); if (!gfs2_quotad_cachep) goto fail_cachep6; gfs2_qadata_cachep = kmem_cache_create("gfs2_qadata", sizeof(struct gfs2_qadata), 0, 0, NULL); if (!gfs2_qadata_cachep) goto fail_cachep7; gfs2_trans_cachep = kmem_cache_create("gfs2_trans", sizeof(struct gfs2_trans), 0, 0, NULL); if (!gfs2_trans_cachep) goto fail_cachep8; error = gfs2_qd_shrinker_init(); if (error) goto fail_shrinker; error = -ENOMEM; gfs2_recovery_wq = alloc_workqueue("gfs2_recovery", WQ_MEM_RECLAIM | WQ_FREEZABLE | WQ_PERCPU, 0); if (!gfs2_recovery_wq) goto fail_wq1; gfs2_control_wq = alloc_workqueue("gfs2_control", WQ_UNBOUND | WQ_FREEZABLE, 0); if (!gfs2_control_wq) goto fail_wq2; gfs2_freeze_wq = alloc_workqueue("gfs2_freeze", WQ_PERCPU, 0); if (!gfs2_freeze_wq) goto fail_wq3; gfs2_page_pool = mempool_create_page_pool(64, 0); if (!gfs2_page_pool) goto fail_mempool; gfs2_register_debugfs(); error = register_filesystem(&gfs2_fs_type); if (error) goto fail_fs1; error = register_filesystem(&gfs2meta_fs_type); if (error) goto fail_fs2; pr_info("GFS2 installed\n"); return 0; fail_fs2: unregister_filesystem(&gfs2_fs_type); fail_fs1: mempool_destroy(gfs2_page_pool); fail_mempool: destroy_workqueue(gfs2_freeze_wq); fail_wq3: destroy_workqueue(gfs2_control_wq); fail_wq2: destroy_workqueue(gfs2_recovery_wq); fail_wq1: gfs2_qd_shrinker_exit(); fail_shrinker: kmem_cache_destroy(gfs2_trans_cachep); fail_cachep8: kmem_cache_destroy(gfs2_qadata_cachep); fail_cachep7: kmem_cache_destroy(gfs2_quotad_cachep); fail_cachep6: kmem_cache_destroy(gfs2_rgrpd_cachep); fail_cachep5: kmem_cache_destroy(gfs2_bufdata_cachep); fail_cachep4: kmem_cache_destroy(gfs2_inode_cachep); fail_cachep3: kmem_cache_destroy(gfs2_glock_aspace_cachep); fail_cachep2: kmem_cache_destroy(gfs2_glock_cachep); fail_cachep1: gfs2_glock_exit(); fail_glock: list_lru_destroy(&gfs2_qd_lru); fail_lru: gfs2_sys_uninit(); return error; } /** * exit_gfs2_fs - Unregister the file system * */ static void __exit exit_gfs2_fs(void) { gfs2_qd_shrinker_exit(); gfs2_glock_exit(); gfs2_unregister_debugfs(); unregister_filesystem(&gfs2_fs_type); unregister_filesystem(&gfs2meta_fs_type); destroy_workqueue(gfs2_recovery_wq); destroy_workqueue(gfs2_control_wq); destroy_workqueue(gfs2_freeze_wq); list_lru_destroy(&gfs2_qd_lru); rcu_barrier(); mempool_destroy(gfs2_page_pool); kmem_cache_destroy(gfs2_trans_cachep); kmem_cache_destroy(gfs2_qadata_cachep); kmem_cache_destroy(gfs2_quotad_cachep); kmem_cache_destroy(gfs2_rgrpd_cachep); kmem_cache_destroy(gfs2_bufdata_cachep); kmem_cache_destroy(gfs2_inode_cachep); kmem_cache_destroy(gfs2_glock_aspace_cachep); kmem_cache_destroy(gfs2_glock_cachep); gfs2_sys_uninit(); } MODULE_DESCRIPTION("Global File System"); MODULE_AUTHOR("Red Hat, Inc."); MODULE_LICENSE("GPL"); module_init(init_gfs2_fs); module_exit(exit_gfs2_fs); |
| 2 1 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 | // SPDX-License-Identifier: GPL-2.0-only #include <linux/phy.h> #include <linux/phylib_stubs.h> #include "netlink.h" #include "common.h" #include "bitset.h" struct stats_req_info { struct ethnl_req_info base; DECLARE_BITMAP(stat_mask, __ETHTOOL_STATS_CNT); enum ethtool_mac_stats_src src; }; #define STATS_REQINFO(__req_base) \ container_of(__req_base, struct stats_req_info, base) struct stats_reply_data { struct ethnl_reply_data base; struct_group(stats, struct ethtool_eth_phy_stats phy_stats; struct ethtool_eth_mac_stats mac_stats; struct ethtool_eth_ctrl_stats ctrl_stats; struct ethtool_rmon_stats rmon_stats; struct ethtool_phy_stats phydev_stats; ); const struct ethtool_rmon_hist_range *rmon_ranges; }; #define STATS_REPDATA(__reply_base) \ container_of(__reply_base, struct stats_reply_data, base) const char stats_std_names[__ETHTOOL_STATS_CNT][ETH_GSTRING_LEN] = { [ETHTOOL_STATS_ETH_PHY] = "eth-phy", [ETHTOOL_STATS_ETH_MAC] = "eth-mac", [ETHTOOL_STATS_ETH_CTRL] = "eth-ctrl", [ETHTOOL_STATS_RMON] = "rmon", [ETHTOOL_STATS_PHY] = "phydev", }; const char stats_eth_phy_names[__ETHTOOL_A_STATS_ETH_PHY_CNT][ETH_GSTRING_LEN] = { [ETHTOOL_A_STATS_ETH_PHY_5_SYM_ERR] = "SymbolErrorDuringCarrier", }; const char stats_eth_mac_names[__ETHTOOL_A_STATS_ETH_MAC_CNT][ETH_GSTRING_LEN] = { [ETHTOOL_A_STATS_ETH_MAC_2_TX_PKT] = "FramesTransmittedOK", [ETHTOOL_A_STATS_ETH_MAC_3_SINGLE_COL] = "SingleCollisionFrames", [ETHTOOL_A_STATS_ETH_MAC_4_MULTI_COL] = "MultipleCollisionFrames", [ETHTOOL_A_STATS_ETH_MAC_5_RX_PKT] = "FramesReceivedOK", [ETHTOOL_A_STATS_ETH_MAC_6_FCS_ERR] = "FrameCheckSequenceErrors", [ETHTOOL_A_STATS_ETH_MAC_7_ALIGN_ERR] = "AlignmentErrors", [ETHTOOL_A_STATS_ETH_MAC_8_TX_BYTES] = "OctetsTransmittedOK", [ETHTOOL_A_STATS_ETH_MAC_9_TX_DEFER] = "FramesWithDeferredXmissions", [ETHTOOL_A_STATS_ETH_MAC_10_LATE_COL] = "LateCollisions", [ETHTOOL_A_STATS_ETH_MAC_11_XS_COL] = "FramesAbortedDueToXSColls", [ETHTOOL_A_STATS_ETH_MAC_12_TX_INT_ERR] = "FramesLostDueToIntMACXmitError", [ETHTOOL_A_STATS_ETH_MAC_13_CS_ERR] = "CarrierSenseErrors", [ETHTOOL_A_STATS_ETH_MAC_14_RX_BYTES] = "OctetsReceivedOK", [ETHTOOL_A_STATS_ETH_MAC_15_RX_INT_ERR] = "FramesLostDueToIntMACRcvError", [ETHTOOL_A_STATS_ETH_MAC_18_TX_MCAST] = "MulticastFramesXmittedOK", [ETHTOOL_A_STATS_ETH_MAC_19_TX_BCAST] = "BroadcastFramesXmittedOK", [ETHTOOL_A_STATS_ETH_MAC_20_XS_DEFER] = "FramesWithExcessiveDeferral", [ETHTOOL_A_STATS_ETH_MAC_21_RX_MCAST] = "MulticastFramesReceivedOK", [ETHTOOL_A_STATS_ETH_MAC_22_RX_BCAST] = "BroadcastFramesReceivedOK", [ETHTOOL_A_STATS_ETH_MAC_23_IR_LEN_ERR] = "InRangeLengthErrors", [ETHTOOL_A_STATS_ETH_MAC_24_OOR_LEN] = "OutOfRangeLengthField", [ETHTOOL_A_STATS_ETH_MAC_25_TOO_LONG_ERR] = "FrameTooLongErrors", }; const char stats_eth_ctrl_names[__ETHTOOL_A_STATS_ETH_CTRL_CNT][ETH_GSTRING_LEN] = { [ETHTOOL_A_STATS_ETH_CTRL_3_TX] = "MACControlFramesTransmitted", [ETHTOOL_A_STATS_ETH_CTRL_4_RX] = "MACControlFramesReceived", [ETHTOOL_A_STATS_ETH_CTRL_5_RX_UNSUP] = "UnsupportedOpcodesReceived", }; const char stats_rmon_names[__ETHTOOL_A_STATS_RMON_CNT][ETH_GSTRING_LEN] = { [ETHTOOL_A_STATS_RMON_UNDERSIZE] = "etherStatsUndersizePkts", [ETHTOOL_A_STATS_RMON_OVERSIZE] = "etherStatsOversizePkts", [ETHTOOL_A_STATS_RMON_FRAG] = "etherStatsFragments", [ETHTOOL_A_STATS_RMON_JABBER] = "etherStatsJabbers", }; const char stats_phy_names[__ETHTOOL_A_STATS_PHY_CNT][ETH_GSTRING_LEN] = { [ETHTOOL_A_STATS_PHY_RX_PKTS] = "RxFrames", [ETHTOOL_A_STATS_PHY_RX_BYTES] = "RxOctets", [ETHTOOL_A_STATS_PHY_RX_ERRORS] = "RxErrors", [ETHTOOL_A_STATS_PHY_TX_PKTS] = "TxFrames", [ETHTOOL_A_STATS_PHY_TX_BYTES] = "TxOctets", [ETHTOOL_A_STATS_PHY_TX_ERRORS] = "TxErrors", }; const struct nla_policy ethnl_stats_get_policy[ETHTOOL_A_STATS_SRC + 1] = { [ETHTOOL_A_STATS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), [ETHTOOL_A_STATS_GROUPS] = { .type = NLA_NESTED }, [ETHTOOL_A_STATS_SRC] = NLA_POLICY_MAX(NLA_U32, ETHTOOL_MAC_STATS_SRC_PMAC), }; static int stats_parse_request(struct ethnl_req_info *req_base, struct nlattr **tb, struct netlink_ext_ack *extack) { enum ethtool_mac_stats_src src = ETHTOOL_MAC_STATS_SRC_AGGREGATE; struct stats_req_info *req_info = STATS_REQINFO(req_base); bool mod = false; int err; err = ethnl_update_bitset(req_info->stat_mask, __ETHTOOL_STATS_CNT, tb[ETHTOOL_A_STATS_GROUPS], stats_std_names, extack, &mod); if (err) return err; if (!mod) { NL_SET_ERR_MSG(extack, "no stats requested"); return -EINVAL; } if (tb[ETHTOOL_A_STATS_SRC]) src = nla_get_u32(tb[ETHTOOL_A_STATS_SRC]); req_info->src = src; return 0; } static int stats_prepare_data(const struct ethnl_req_info *req_base, struct ethnl_reply_data *reply_base, const struct genl_info *info) { const struct stats_req_info *req_info = STATS_REQINFO(req_base); struct stats_reply_data *data = STATS_REPDATA(reply_base); enum ethtool_mac_stats_src src = req_info->src; struct net_device *dev = reply_base->dev; struct nlattr **tb = info->attrs; struct phy_device *phydev; int ret; phydev = ethnl_req_get_phydev(req_base, tb, ETHTOOL_A_STATS_HEADER, info->extack); if (IS_ERR(phydev)) return PTR_ERR(phydev); ret = ethnl_ops_begin(dev); if (ret < 0) return ret; if ((src == ETHTOOL_MAC_STATS_SRC_EMAC || src == ETHTOOL_MAC_STATS_SRC_PMAC) && !__ethtool_dev_mm_supported(dev)) { NL_SET_ERR_MSG_MOD(info->extack, "Device does not support MAC merge layer"); ethnl_ops_complete(dev); return -EOPNOTSUPP; } /* Mark all stats as unset (see ETHTOOL_STAT_NOT_SET) to prevent them * from being reported to user space in case driver did not set them. */ memset(&data->stats, 0xff, sizeof(data->stats)); data->phy_stats.src = src; data->mac_stats.src = src; data->ctrl_stats.src = src; data->rmon_stats.src = src; if ((test_bit(ETHTOOL_STATS_PHY, req_info->stat_mask) || test_bit(ETHTOOL_STATS_ETH_PHY, req_info->stat_mask)) && src == ETHTOOL_MAC_STATS_SRC_AGGREGATE) { if (phydev) phy_ethtool_get_phy_stats(phydev, &data->phy_stats, &data->phydev_stats); } if (test_bit(ETHTOOL_STATS_ETH_PHY, req_info->stat_mask) && dev->ethtool_ops->get_eth_phy_stats) dev->ethtool_ops->get_eth_phy_stats(dev, &data->phy_stats); if (test_bit(ETHTOOL_STATS_ETH_MAC, req_info->stat_mask) && dev->ethtool_ops->get_eth_mac_stats) dev->ethtool_ops->get_eth_mac_stats(dev, &data->mac_stats); if (test_bit(ETHTOOL_STATS_ETH_CTRL, req_info->stat_mask) && dev->ethtool_ops->get_eth_ctrl_stats) dev->ethtool_ops->get_eth_ctrl_stats(dev, &data->ctrl_stats); if (test_bit(ETHTOOL_STATS_RMON, req_info->stat_mask) && dev->ethtool_ops->get_rmon_stats) dev->ethtool_ops->get_rmon_stats(dev, &data->rmon_stats, &data->rmon_ranges); ethnl_ops_complete(dev); return 0; } static int stats_reply_size(const struct ethnl_req_info *req_base, const struct ethnl_reply_data *reply_base) { const struct stats_req_info *req_info = STATS_REQINFO(req_base); unsigned int n_grps = 0, n_stats = 0; int len = 0; len += nla_total_size(sizeof(u32)); /* _STATS_SRC */ if (test_bit(ETHTOOL_STATS_ETH_PHY, req_info->stat_mask)) { n_stats += sizeof(struct ethtool_eth_phy_stats) / sizeof(u64); n_grps++; } if (test_bit(ETHTOOL_STATS_ETH_MAC, req_info->stat_mask)) { n_stats += sizeof(struct ethtool_eth_mac_stats) / sizeof(u64); n_grps++; } if (test_bit(ETHTOOL_STATS_ETH_CTRL, req_info->stat_mask)) { n_stats += sizeof(struct ethtool_eth_ctrl_stats) / sizeof(u64); n_grps++; } if (test_bit(ETHTOOL_STATS_RMON, req_info->stat_mask)) { n_stats += sizeof(struct ethtool_rmon_stats) / sizeof(u64); n_grps++; /* Above includes the space for _A_STATS_GRP_HIST_VALs */ len += (nla_total_size(0) + /* _A_STATS_GRP_HIST */ nla_total_size(4) + /* _A_STATS_GRP_HIST_BKT_LOW */ nla_total_size(4)) * /* _A_STATS_GRP_HIST_BKT_HI */ ETHTOOL_RMON_HIST_MAX * 2; } if (test_bit(ETHTOOL_STATS_PHY, req_info->stat_mask)) { n_stats += sizeof(struct ethtool_phy_stats) / sizeof(u64); n_grps++; } len += n_grps * (nla_total_size(0) + /* _A_STATS_GRP */ nla_total_size(4) + /* _A_STATS_GRP_ID */ nla_total_size(4)); /* _A_STATS_GRP_SS_ID */ len += n_stats * (nla_total_size(0) + /* _A_STATS_GRP_STAT */ nla_total_size_64bit(sizeof(u64))); return len; } static int stat_put(struct sk_buff *skb, u16 attrtype, u64 val) { struct nlattr *nest; int ret; if (val == ETHTOOL_STAT_NOT_SET) return 0; /* We want to start stats attr types from 0, so we don't have a type * for pad inside ETHTOOL_A_STATS_GRP_STAT. Pad things on the outside * of ETHTOOL_A_STATS_GRP_STAT. Since we're one nest away from the * actual attr we're 4B off - nla_need_padding_for_64bit() & co. * can't be used. */ #ifndef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS if (!IS_ALIGNED((unsigned long)skb_tail_pointer(skb), 8)) if (!nla_reserve(skb, ETHTOOL_A_STATS_GRP_PAD, 0)) return -EMSGSIZE; #endif nest = nla_nest_start(skb, ETHTOOL_A_STATS_GRP_STAT); if (!nest) return -EMSGSIZE; ret = nla_put_u64_64bit(skb, attrtype, val, -1 /* not used */); if (ret) { nla_nest_cancel(skb, nest); return ret; } nla_nest_end(skb, nest); return 0; } static int stats_put_phy_stats(struct sk_buff *skb, const struct stats_reply_data *data) { if (stat_put(skb, ETHTOOL_A_STATS_ETH_PHY_5_SYM_ERR, data->phy_stats.SymbolErrorDuringCarrier)) return -EMSGSIZE; return 0; } static int stats_put_phydev_stats(struct sk_buff *skb, const struct stats_reply_data *data) { if (stat_put(skb, ETHTOOL_A_STATS_PHY_RX_PKTS, data->phydev_stats.rx_packets) || stat_put(skb, ETHTOOL_A_STATS_PHY_RX_BYTES, data->phydev_stats.rx_bytes) || stat_put(skb, ETHTOOL_A_STATS_PHY_RX_ERRORS, data->phydev_stats.rx_errors) || stat_put(skb, ETHTOOL_A_STATS_PHY_TX_PKTS, data->phydev_stats.tx_packets) || stat_put(skb, ETHTOOL_A_STATS_PHY_TX_BYTES, data->phydev_stats.tx_bytes) || stat_put(skb, ETHTOOL_A_STATS_PHY_TX_ERRORS, data->phydev_stats.tx_errors)) return -EMSGSIZE; return 0; } static int stats_put_mac_stats(struct sk_buff *skb, const struct stats_reply_data *data) { if (stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_2_TX_PKT, data->mac_stats.FramesTransmittedOK) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_3_SINGLE_COL, data->mac_stats.SingleCollisionFrames) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_4_MULTI_COL, data->mac_stats.MultipleCollisionFrames) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_5_RX_PKT, data->mac_stats.FramesReceivedOK) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_6_FCS_ERR, data->mac_stats.FrameCheckSequenceErrors) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_7_ALIGN_ERR, data->mac_stats.AlignmentErrors) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_8_TX_BYTES, data->mac_stats.OctetsTransmittedOK) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_9_TX_DEFER, data->mac_stats.FramesWithDeferredXmissions) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_10_LATE_COL, data->mac_stats.LateCollisions) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_11_XS_COL, data->mac_stats.FramesAbortedDueToXSColls) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_12_TX_INT_ERR, data->mac_stats.FramesLostDueToIntMACXmitError) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_13_CS_ERR, data->mac_stats.CarrierSenseErrors) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_14_RX_BYTES, data->mac_stats.OctetsReceivedOK) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_15_RX_INT_ERR, data->mac_stats.FramesLostDueToIntMACRcvError) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_18_TX_MCAST, data->mac_stats.MulticastFramesXmittedOK) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_19_TX_BCAST, data->mac_stats.BroadcastFramesXmittedOK) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_20_XS_DEFER, data->mac_stats.FramesWithExcessiveDeferral) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_21_RX_MCAST, data->mac_stats.MulticastFramesReceivedOK) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_22_RX_BCAST, data->mac_stats.BroadcastFramesReceivedOK) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_23_IR_LEN_ERR, data->mac_stats.InRangeLengthErrors) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_24_OOR_LEN, data->mac_stats.OutOfRangeLengthField) || stat_put(skb, ETHTOOL_A_STATS_ETH_MAC_25_TOO_LONG_ERR, data->mac_stats.FrameTooLongErrors)) return -EMSGSIZE; return 0; } static int stats_put_ctrl_stats(struct sk_buff *skb, const struct stats_reply_data *data) { if (stat_put(skb, ETHTOOL_A_STATS_ETH_CTRL_3_TX, data->ctrl_stats.MACControlFramesTransmitted) || stat_put(skb, ETHTOOL_A_STATS_ETH_CTRL_4_RX, data->ctrl_stats.MACControlFramesReceived) || stat_put(skb, ETHTOOL_A_STATS_ETH_CTRL_5_RX_UNSUP, data->ctrl_stats.UnsupportedOpcodesReceived)) return -EMSGSIZE; return 0; } static int stats_put_rmon_hist(struct sk_buff *skb, u32 attr, const u64 *hist, const struct ethtool_rmon_hist_range *ranges) { struct nlattr *nest; int i; if (!ranges) return 0; for (i = 0; i < ETHTOOL_RMON_HIST_MAX; i++) { if (!ranges[i].low && !ranges[i].high) break; if (hist[i] == ETHTOOL_STAT_NOT_SET) continue; nest = nla_nest_start(skb, attr); if (!nest) return -EMSGSIZE; if (nla_put_u32(skb, ETHTOOL_A_STATS_GRP_HIST_BKT_LOW, ranges[i].low) || nla_put_u32(skb, ETHTOOL_A_STATS_GRP_HIST_BKT_HI, ranges[i].high) || nla_put_u64_64bit(skb, ETHTOOL_A_STATS_GRP_HIST_VAL, hist[i], ETHTOOL_A_STATS_GRP_PAD)) goto err_cancel_hist; nla_nest_end(skb, nest); } return 0; err_cancel_hist: nla_nest_cancel(skb, nest); return -EMSGSIZE; } static int stats_put_rmon_stats(struct sk_buff *skb, const struct stats_reply_data *data) { if (stats_put_rmon_hist(skb, ETHTOOL_A_STATS_GRP_HIST_RX, data->rmon_stats.hist, data->rmon_ranges) || stats_put_rmon_hist(skb, ETHTOOL_A_STATS_GRP_HIST_TX, data->rmon_stats.hist_tx, data->rmon_ranges)) return -EMSGSIZE; if (stat_put(skb, ETHTOOL_A_STATS_RMON_UNDERSIZE, data->rmon_stats.undersize_pkts) || stat_put(skb, ETHTOOL_A_STATS_RMON_OVERSIZE, data->rmon_stats.oversize_pkts) || stat_put(skb, ETHTOOL_A_STATS_RMON_FRAG, data->rmon_stats.fragments) || stat_put(skb, ETHTOOL_A_STATS_RMON_JABBER, data->rmon_stats.jabbers)) return -EMSGSIZE; return 0; } static int stats_put_stats(struct sk_buff *skb, const struct stats_reply_data *data, u32 id, u32 ss_id, int (*cb)(struct sk_buff *skb, const struct stats_reply_data *data)) { struct nlattr *nest; nest = nla_nest_start(skb, ETHTOOL_A_STATS_GRP); if (!nest) return -EMSGSIZE; if (nla_put_u32(skb, ETHTOOL_A_STATS_GRP_ID, id) || nla_put_u32(skb, ETHTOOL_A_STATS_GRP_SS_ID, ss_id)) goto err_cancel; if (cb(skb, data)) goto err_cancel; nla_nest_end(skb, nest); return 0; err_cancel: nla_nest_cancel(skb, nest); return -EMSGSIZE; } static int stats_fill_reply(struct sk_buff *skb, const struct ethnl_req_info *req_base, const struct ethnl_reply_data *reply_base) { const struct stats_req_info *req_info = STATS_REQINFO(req_base); const struct stats_reply_data *data = STATS_REPDATA(reply_base); int ret = 0; if (nla_put_u32(skb, ETHTOOL_A_STATS_SRC, req_info->src)) return -EMSGSIZE; if (!ret && test_bit(ETHTOOL_STATS_ETH_PHY, req_info->stat_mask)) ret = stats_put_stats(skb, data, ETHTOOL_STATS_ETH_PHY, ETH_SS_STATS_ETH_PHY, stats_put_phy_stats); if (!ret && test_bit(ETHTOOL_STATS_ETH_MAC, req_info->stat_mask)) ret = stats_put_stats(skb, data, ETHTOOL_STATS_ETH_MAC, ETH_SS_STATS_ETH_MAC, stats_put_mac_stats); if (!ret && test_bit(ETHTOOL_STATS_ETH_CTRL, req_info->stat_mask)) ret = stats_put_stats(skb, data, ETHTOOL_STATS_ETH_CTRL, ETH_SS_STATS_ETH_CTRL, stats_put_ctrl_stats); if (!ret && test_bit(ETHTOOL_STATS_RMON, req_info->stat_mask)) ret = stats_put_stats(skb, data, ETHTOOL_STATS_RMON, ETH_SS_STATS_RMON, stats_put_rmon_stats); if (!ret && test_bit(ETHTOOL_STATS_PHY, req_info->stat_mask)) ret = stats_put_stats(skb, data, ETHTOOL_STATS_PHY, ETH_SS_STATS_PHY, stats_put_phydev_stats); return ret; } const struct ethnl_request_ops ethnl_stats_request_ops = { .request_cmd = ETHTOOL_MSG_STATS_GET, .reply_cmd = ETHTOOL_MSG_STATS_GET_REPLY, .hdr_attr = ETHTOOL_A_STATS_HEADER, .req_info_size = sizeof(struct stats_req_info), .reply_data_size = sizeof(struct stats_reply_data), .parse_request = stats_parse_request, .prepare_data = stats_prepare_data, .reply_size = stats_reply_size, .fill_reply = stats_fill_reply, }; static u64 ethtool_stats_sum(u64 a, u64 b) { if (a == ETHTOOL_STAT_NOT_SET) return b; if (b == ETHTOOL_STAT_NOT_SET) return a; return a + b; } /* Avoid modifying the aggregation procedure every time a new counter is added * by treating the structures as an array of u64 statistics. */ static void ethtool_aggregate_stats(void *aggr_stats, const void *emac_stats, const void *pmac_stats, size_t stats_size, size_t stats_offset) { size_t num_stats = stats_size / sizeof(u64); const u64 *s1 = emac_stats + stats_offset; const u64 *s2 = pmac_stats + stats_offset; u64 *s = aggr_stats + stats_offset; int i; for (i = 0; i < num_stats; i++) s[i] = ethtool_stats_sum(s1[i], s2[i]); } void ethtool_aggregate_mac_stats(struct net_device *dev, struct ethtool_eth_mac_stats *mac_stats) { const struct ethtool_ops *ops = dev->ethtool_ops; struct ethtool_eth_mac_stats pmac, emac; memset(&emac, 0xff, sizeof(emac)); memset(&pmac, 0xff, sizeof(pmac)); emac.src = ETHTOOL_MAC_STATS_SRC_EMAC; pmac.src = ETHTOOL_MAC_STATS_SRC_PMAC; ops->get_eth_mac_stats(dev, &emac); ops->get_eth_mac_stats(dev, &pmac); ethtool_aggregate_stats(mac_stats, &emac, &pmac, sizeof(mac_stats->stats), offsetof(struct ethtool_eth_mac_stats, stats)); } EXPORT_SYMBOL(ethtool_aggregate_mac_stats); void ethtool_aggregate_phy_stats(struct net_device *dev, struct ethtool_eth_phy_stats *phy_stats) { const struct ethtool_ops *ops = dev->ethtool_ops; struct ethtool_eth_phy_stats pmac, emac; memset(&emac, 0xff, sizeof(emac)); memset(&pmac, 0xff, sizeof(pmac)); emac.src = ETHTOOL_MAC_STATS_SRC_EMAC; pmac.src = ETHTOOL_MAC_STATS_SRC_PMAC; ops->get_eth_phy_stats(dev, &emac); ops->get_eth_phy_stats(dev, &pmac); ethtool_aggregate_stats(phy_stats, &emac, &pmac, sizeof(phy_stats->stats), offsetof(struct ethtool_eth_phy_stats, stats)); } EXPORT_SYMBOL(ethtool_aggregate_phy_stats); void ethtool_aggregate_ctrl_stats(struct net_device *dev, struct ethtool_eth_ctrl_stats *ctrl_stats) { const struct ethtool_ops *ops = dev->ethtool_ops; struct ethtool_eth_ctrl_stats pmac, emac; memset(&emac, 0xff, sizeof(emac)); memset(&pmac, 0xff, sizeof(pmac)); emac.src = ETHTOOL_MAC_STATS_SRC_EMAC; pmac.src = ETHTOOL_MAC_STATS_SRC_PMAC; ops->get_eth_ctrl_stats(dev, &emac); ops->get_eth_ctrl_stats(dev, &pmac); ethtool_aggregate_stats(ctrl_stats, &emac, &pmac, sizeof(ctrl_stats->stats), offsetof(struct ethtool_eth_ctrl_stats, stats)); } EXPORT_SYMBOL(ethtool_aggregate_ctrl_stats); void ethtool_aggregate_pause_stats(struct net_device *dev, struct ethtool_pause_stats *pause_stats) { const struct ethtool_ops *ops = dev->ethtool_ops; struct ethtool_pause_stats pmac, emac; memset(&emac, 0xff, sizeof(emac)); memset(&pmac, 0xff, sizeof(pmac)); emac.src = ETHTOOL_MAC_STATS_SRC_EMAC; pmac.src = ETHTOOL_MAC_STATS_SRC_PMAC; ops->get_pause_stats(dev, &emac); ops->get_pause_stats(dev, &pmac); ethtool_aggregate_stats(pause_stats, &emac, &pmac, sizeof(pause_stats->stats), offsetof(struct ethtool_pause_stats, stats)); } EXPORT_SYMBOL(ethtool_aggregate_pause_stats); void ethtool_aggregate_rmon_stats(struct net_device *dev, struct ethtool_rmon_stats *rmon_stats) { const struct ethtool_ops *ops = dev->ethtool_ops; const struct ethtool_rmon_hist_range *dummy; struct ethtool_rmon_stats pmac, emac; memset(&emac, 0xff, sizeof(emac)); memset(&pmac, 0xff, sizeof(pmac)); emac.src = ETHTOOL_MAC_STATS_SRC_EMAC; pmac.src = ETHTOOL_MAC_STATS_SRC_PMAC; ops->get_rmon_stats(dev, &emac, &dummy); ops->get_rmon_stats(dev, &pmac, &dummy); ethtool_aggregate_stats(rmon_stats, &emac, &pmac, sizeof(rmon_stats->stats), offsetof(struct ethtool_rmon_stats, stats)); } EXPORT_SYMBOL(ethtool_aggregate_rmon_stats); |
| 4 4 2 2 4 2 1 1 4 4 1 1 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 | // SPDX-License-Identifier: GPL-2.0+ /* * TI 3410/5052 USB Serial Driver * * Copyright (C) 2004 Texas Instruments * * This driver is based on the Linux io_ti driver, which is * Copyright (C) 2000-2002 Inside Out Networks * Copyright (C) 2001-2002 Greg Kroah-Hartman * * For questions or problems with this driver, contact Texas Instruments * technical support, or Al Borchers <alborchers@steinerpoint.com>, or * Peter Berger <pberger@brimson.com>. */ #include <linux/kernel.h> #include <linux/errno.h> #include <linux/firmware.h> #include <linux/slab.h> #include <linux/tty.h> #include <linux/tty_driver.h> #include <linux/tty_flip.h> #include <linux/module.h> #include <linux/spinlock.h> #include <linux/ioctl.h> #include <linux/serial.h> #include <linux/kfifo.h> #include <linux/mutex.h> #include <linux/uaccess.h> #include <linux/usb.h> #include <linux/usb/serial.h> /* Configuration ids */ #define TI_BOOT_CONFIG 1 #define TI_ACTIVE_CONFIG 2 /* Vendor and product ids */ #define TI_VENDOR_ID 0x0451 #define IBM_VENDOR_ID 0x04b3 #define STARTECH_VENDOR_ID 0x14b0 #define TI_3410_PRODUCT_ID 0x3410 #define IBM_4543_PRODUCT_ID 0x4543 #define IBM_454B_PRODUCT_ID 0x454b #define IBM_454C_PRODUCT_ID 0x454c #define TI_3410_EZ430_ID 0xF430 /* TI ez430 development tool */ #define TI_5052_BOOT_PRODUCT_ID 0x5052 /* no EEPROM, no firmware */ #define TI_5152_BOOT_PRODUCT_ID 0x5152 /* no EEPROM, no firmware */ #define TI_5052_EEPROM_PRODUCT_ID 0x505A /* EEPROM, no firmware */ #define TI_5052_FIRMWARE_PRODUCT_ID 0x505F /* firmware is running */ #define FRI2_PRODUCT_ID 0x5053 /* Fish River Island II */ /* Multi-Tech vendor and product ids */ #define MTS_VENDOR_ID 0x06E0 #define MTS_GSM_NO_FW_PRODUCT_ID 0xF108 #define MTS_CDMA_NO_FW_PRODUCT_ID 0xF109 #define MTS_CDMA_PRODUCT_ID 0xF110 #define MTS_GSM_PRODUCT_ID 0xF111 #define MTS_EDGE_PRODUCT_ID 0xF112 #define MTS_MT9234MU_PRODUCT_ID 0xF114 #define MTS_MT9234ZBA_PRODUCT_ID 0xF115 #define MTS_MT9234ZBAOLD_PRODUCT_ID 0x0319 /* Abbott Diabetics vendor and product ids */ #define ABBOTT_VENDOR_ID 0x1a61 #define ABBOTT_STEREO_PLUG_ID 0x3410 #define ABBOTT_PRODUCT_ID ABBOTT_STEREO_PLUG_ID #define ABBOTT_STRIP_PORT_ID 0x3420 /* Honeywell vendor and product IDs */ #define HONEYWELL_VENDOR_ID 0x10ac #define HONEYWELL_HGI80_PRODUCT_ID 0x0102 /* Honeywell HGI80 */ /* Moxa UPORT 11x0 vendor and product IDs */ #define MXU1_VENDOR_ID 0x110a #define MXU1_1110_PRODUCT_ID 0x1110 #define MXU1_1130_PRODUCT_ID 0x1130 #define MXU1_1150_PRODUCT_ID 0x1150 #define MXU1_1151_PRODUCT_ID 0x1151 #define MXU1_1131_PRODUCT_ID 0x1131 /* Commands */ #define TI_GET_VERSION 0x01 #define TI_GET_PORT_STATUS 0x02 #define TI_GET_PORT_DEV_INFO 0x03 #define TI_GET_CONFIG 0x04 #define TI_SET_CONFIG 0x05 #define TI_OPEN_PORT 0x06 #define TI_CLOSE_PORT 0x07 #define TI_START_PORT 0x08 #define TI_STOP_PORT 0x09 #define TI_TEST_PORT 0x0A #define TI_PURGE_PORT 0x0B #define TI_RESET_EXT_DEVICE 0x0C #define TI_WRITE_DATA 0x80 #define TI_READ_DATA 0x81 #define TI_REQ_TYPE_CLASS 0x82 /* Module identifiers */ #define TI_I2C_PORT 0x01 #define TI_IEEE1284_PORT 0x02 #define TI_UART1_PORT 0x03 #define TI_UART2_PORT 0x04 #define TI_RAM_PORT 0x05 /* Modem status */ #define TI_MSR_DELTA_CTS 0x01 #define TI_MSR_DELTA_DSR 0x02 #define TI_MSR_DELTA_RI 0x04 #define TI_MSR_DELTA_CD 0x08 #define TI_MSR_CTS 0x10 #define TI_MSR_DSR 0x20 #define TI_MSR_RI 0x40 #define TI_MSR_CD 0x80 #define TI_MSR_DELTA_MASK 0x0F #define TI_MSR_MASK 0xF0 /* Line status */ #define TI_LSR_OVERRUN_ERROR 0x01 #define TI_LSR_PARITY_ERROR 0x02 #define TI_LSR_FRAMING_ERROR 0x04 #define TI_LSR_BREAK 0x08 #define TI_LSR_ERROR 0x0F #define TI_LSR_RX_FULL 0x10 #define TI_LSR_TX_EMPTY 0x20 #define TI_LSR_TX_EMPTY_BOTH 0x40 /* Line control */ #define TI_LCR_BREAK 0x40 /* Modem control */ #define TI_MCR_LOOP 0x04 #define TI_MCR_DTR 0x10 #define TI_MCR_RTS 0x20 /* Mask settings */ #define TI_UART_ENABLE_RTS_IN 0x0001 #define TI_UART_DISABLE_RTS 0x0002 #define TI_UART_ENABLE_PARITY_CHECKING 0x0008 #define TI_UART_ENABLE_DSR_OUT 0x0010 #define TI_UART_ENABLE_CTS_OUT 0x0020 #define TI_UART_ENABLE_X_OUT 0x0040 #define TI_UART_ENABLE_XA_OUT 0x0080 #define TI_UART_ENABLE_X_IN 0x0100 #define TI_UART_ENABLE_DTR_IN 0x0800 #define TI_UART_DISABLE_DTR 0x1000 #define TI_UART_ENABLE_MS_INTS 0x2000 #define TI_UART_ENABLE_AUTO_START_DMA 0x4000 /* Parity */ #define TI_UART_NO_PARITY 0x00 #define TI_UART_ODD_PARITY 0x01 #define TI_UART_EVEN_PARITY 0x02 #define TI_UART_MARK_PARITY 0x03 #define TI_UART_SPACE_PARITY 0x04 /* Stop bits */ #define TI_UART_1_STOP_BITS 0x00 #define TI_UART_1_5_STOP_BITS 0x01 #define TI_UART_2_STOP_BITS 0x02 /* Bits per character */ #define TI_UART_5_DATA_BITS 0x00 #define TI_UART_6_DATA_BITS 0x01 #define TI_UART_7_DATA_BITS 0x02 #define TI_UART_8_DATA_BITS 0x03 /* 232/485 modes */ #define TI_UART_232 0x00 #define TI_UART_485_RECEIVER_DISABLED 0x01 #define TI_UART_485_RECEIVER_ENABLED 0x02 /* Pipe transfer mode and timeout */ #define TI_PIPE_MODE_CONTINUOUS 0x01 #define TI_PIPE_MODE_MASK 0x03 #define TI_PIPE_TIMEOUT_MASK 0x7C #define TI_PIPE_TIMEOUT_ENABLE 0x80 /* Config struct */ struct ti_uart_config { __be16 wBaudRate; __be16 wFlags; u8 bDataBits; u8 bParity; u8 bStopBits; char cXon; char cXoff; u8 bUartMode; }; /* Get port status */ struct ti_port_status { u8 bCmdCode; u8 bModuleId; u8 bErrorCode; u8 bMSR; u8 bLSR; }; /* Purge modes */ #define TI_PURGE_OUTPUT 0x00 #define TI_PURGE_INPUT 0x80 /* Read/Write data */ #define TI_RW_DATA_ADDR_SFR 0x10 #define TI_RW_DATA_ADDR_IDATA 0x20 #define TI_RW_DATA_ADDR_XDATA 0x30 #define TI_RW_DATA_ADDR_CODE 0x40 #define TI_RW_DATA_ADDR_GPIO 0x50 #define TI_RW_DATA_ADDR_I2C 0x60 #define TI_RW_DATA_ADDR_FLASH 0x70 #define TI_RW_DATA_ADDR_DSP 0x80 #define TI_RW_DATA_UNSPECIFIED 0x00 #define TI_RW_DATA_BYTE 0x01 #define TI_RW_DATA_WORD 0x02 #define TI_RW_DATA_DOUBLE_WORD 0x04 struct ti_write_data_bytes { u8 bAddrType; u8 bDataType; u8 bDataCounter; __be16 wBaseAddrHi; __be16 wBaseAddrLo; u8 bData[]; } __packed; struct ti_read_data_request { u8 bAddrType; u8 bDataType; u8 bDataCounter; __be16 wBaseAddrHi; __be16 wBaseAddrLo; } __packed; struct ti_read_data_bytes { u8 bCmdCode; u8 bModuleId; u8 bErrorCode; u8 bData[]; }; /* Interrupt struct */ struct ti_interrupt { u8 bICode; u8 bIInfo; }; /* Interrupt codes */ #define TI_CODE_HARDWARE_ERROR 0xFF #define TI_CODE_DATA_ERROR 0x03 #define TI_CODE_MODEM_STATUS 0x04 /* Download firmware max packet size */ #define TI_DOWNLOAD_MAX_PACKET_SIZE 64 /* Firmware image header */ struct ti_firmware_header { __le16 wLength; u8 bCheckSum; } __packed; /* UART addresses */ #define TI_UART1_BASE_ADDR 0xFFA0 /* UART 1 base address */ #define TI_UART2_BASE_ADDR 0xFFB0 /* UART 2 base address */ #define TI_UART_OFFSET_LCR 0x0002 /* UART MCR register offset */ #define TI_UART_OFFSET_MCR 0x0004 /* UART MCR register offset */ #define TI_DRIVER_AUTHOR "Al Borchers <alborchers@steinerpoint.com>" #define TI_DRIVER_DESC "TI USB 3410/5052 Serial Driver" #define TI_FIRMWARE_BUF_SIZE 16284 #define TI_TRANSFER_TIMEOUT 2 /* read urb states */ #define TI_READ_URB_RUNNING 0 #define TI_READ_URB_STOPPING 1 #define TI_READ_URB_STOPPED 2 #define TI_EXTRA_VID_PID_COUNT 5 struct ti_port { int tp_is_open; u8 tp_msr; u8 tp_shadow_mcr; u8 tp_uart_mode; /* 232 or 485 modes */ unsigned int tp_uart_base_addr; struct ti_device *tp_tdev; struct usb_serial_port *tp_port; spinlock_t tp_lock; int tp_read_urb_state; int tp_write_urb_in_use; }; struct ti_device { struct mutex td_open_close_lock; int td_open_port_count; struct usb_serial *td_serial; int td_is_3410; bool td_rs485_only; }; static int ti_startup(struct usb_serial *serial); static void ti_release(struct usb_serial *serial); static int ti_port_probe(struct usb_serial_port *port); static void ti_port_remove(struct usb_serial_port *port); static int ti_open(struct tty_struct *tty, struct usb_serial_port *port); static void ti_close(struct usb_serial_port *port); static int ti_write(struct tty_struct *tty, struct usb_serial_port *port, const unsigned char *data, int count); static unsigned int ti_write_room(struct tty_struct *tty); static unsigned int ti_chars_in_buffer(struct tty_struct *tty); static bool ti_tx_empty(struct usb_serial_port *port); static void ti_throttle(struct tty_struct *tty); static void ti_unthrottle(struct tty_struct *tty); static void ti_set_termios(struct tty_struct *tty, struct usb_serial_port *port, const struct ktermios *old_termios); static int ti_tiocmget(struct tty_struct *tty); static int ti_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear); static int ti_break(struct tty_struct *tty, int break_state); static void ti_interrupt_callback(struct urb *urb); static void ti_bulk_in_callback(struct urb *urb); static void ti_bulk_out_callback(struct urb *urb); static void ti_recv(struct usb_serial_port *port, unsigned char *data, int length); static void ti_send(struct ti_port *tport); static int ti_set_mcr(struct ti_port *tport, unsigned int mcr); static int ti_get_lsr(struct ti_port *tport, u8 *lsr); static void ti_get_serial_info(struct tty_struct *tty, struct serial_struct *ss); static void ti_handle_new_msr(struct ti_port *tport, u8 msr); static void ti_stop_read(struct ti_port *tport, struct tty_struct *tty); static int ti_restart_read(struct ti_port *tport, struct tty_struct *tty); static int ti_command_out_sync(struct usb_device *udev, u8 command, u16 moduleid, u16 value, void *data, int size); static int ti_command_in_sync(struct usb_device *udev, u8 command, u16 moduleid, u16 value, void *data, int size); static int ti_port_cmd_out(struct usb_serial_port *port, u8 command, u16 value, void *data, int size); static int ti_port_cmd_in(struct usb_serial_port *port, u8 command, u16 value, void *data, int size); static int ti_write_byte(struct usb_serial_port *port, struct ti_device *tdev, unsigned long addr, u8 mask, u8 byte); static int ti_download_firmware(struct ti_device *tdev); static const struct usb_device_id ti_id_table_3410[] = { { USB_DEVICE(TI_VENDOR_ID, TI_3410_PRODUCT_ID) }, { USB_DEVICE(TI_VENDOR_ID, TI_3410_EZ430_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_GSM_NO_FW_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_CDMA_NO_FW_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_CDMA_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_GSM_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_EDGE_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_MT9234MU_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_MT9234ZBA_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_MT9234ZBAOLD_PRODUCT_ID) }, { USB_DEVICE(IBM_VENDOR_ID, IBM_4543_PRODUCT_ID) }, { USB_DEVICE(IBM_VENDOR_ID, IBM_454B_PRODUCT_ID) }, { USB_DEVICE(IBM_VENDOR_ID, IBM_454C_PRODUCT_ID) }, { USB_DEVICE(ABBOTT_VENDOR_ID, ABBOTT_STEREO_PLUG_ID) }, { USB_DEVICE(ABBOTT_VENDOR_ID, ABBOTT_STRIP_PORT_ID) }, { USB_DEVICE(TI_VENDOR_ID, FRI2_PRODUCT_ID) }, { USB_DEVICE(HONEYWELL_VENDOR_ID, HONEYWELL_HGI80_PRODUCT_ID) }, { USB_DEVICE(MXU1_VENDOR_ID, MXU1_1110_PRODUCT_ID) }, { USB_DEVICE(MXU1_VENDOR_ID, MXU1_1130_PRODUCT_ID) }, { USB_DEVICE(MXU1_VENDOR_ID, MXU1_1131_PRODUCT_ID) }, { USB_DEVICE(MXU1_VENDOR_ID, MXU1_1150_PRODUCT_ID) }, { USB_DEVICE(MXU1_VENDOR_ID, MXU1_1151_PRODUCT_ID) }, { USB_DEVICE(STARTECH_VENDOR_ID, TI_3410_PRODUCT_ID) }, { } /* terminator */ }; static const struct usb_device_id ti_id_table_5052[] = { { USB_DEVICE(TI_VENDOR_ID, TI_5052_BOOT_PRODUCT_ID) }, { USB_DEVICE(TI_VENDOR_ID, TI_5152_BOOT_PRODUCT_ID) }, { USB_DEVICE(TI_VENDOR_ID, TI_5052_EEPROM_PRODUCT_ID) }, { USB_DEVICE(TI_VENDOR_ID, TI_5052_FIRMWARE_PRODUCT_ID) }, { } }; static const struct usb_device_id ti_id_table_combined[] = { { USB_DEVICE(TI_VENDOR_ID, TI_3410_PRODUCT_ID) }, { USB_DEVICE(TI_VENDOR_ID, TI_3410_EZ430_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_GSM_NO_FW_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_CDMA_NO_FW_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_CDMA_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_GSM_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_EDGE_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_MT9234MU_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_MT9234ZBA_PRODUCT_ID) }, { USB_DEVICE(MTS_VENDOR_ID, MTS_MT9234ZBAOLD_PRODUCT_ID) }, { USB_DEVICE(TI_VENDOR_ID, TI_5052_BOOT_PRODUCT_ID) }, { USB_DEVICE(TI_VENDOR_ID, TI_5152_BOOT_PRODUCT_ID) }, { USB_DEVICE(TI_VENDOR_ID, TI_5052_EEPROM_PRODUCT_ID) }, { USB_DEVICE(TI_VENDOR_ID, TI_5052_FIRMWARE_PRODUCT_ID) }, { USB_DEVICE(IBM_VENDOR_ID, IBM_4543_PRODUCT_ID) }, { USB_DEVICE(IBM_VENDOR_ID, IBM_454B_PRODUCT_ID) }, { USB_DEVICE(IBM_VENDOR_ID, IBM_454C_PRODUCT_ID) }, { USB_DEVICE(ABBOTT_VENDOR_ID, ABBOTT_PRODUCT_ID) }, { USB_DEVICE(ABBOTT_VENDOR_ID, ABBOTT_STRIP_PORT_ID) }, { USB_DEVICE(TI_VENDOR_ID, FRI2_PRODUCT_ID) }, { USB_DEVICE(HONEYWELL_VENDOR_ID, HONEYWELL_HGI80_PRODUCT_ID) }, { USB_DEVICE(MXU1_VENDOR_ID, MXU1_1110_PRODUCT_ID) }, { USB_DEVICE(MXU1_VENDOR_ID, MXU1_1130_PRODUCT_ID) }, { USB_DEVICE(MXU1_VENDOR_ID, MXU1_1131_PRODUCT_ID) }, { USB_DEVICE(MXU1_VENDOR_ID, MXU1_1150_PRODUCT_ID) }, { USB_DEVICE(MXU1_VENDOR_ID, MXU1_1151_PRODUCT_ID) }, { USB_DEVICE(STARTECH_VENDOR_ID, TI_3410_PRODUCT_ID) }, { } /* terminator */ }; static struct usb_serial_driver ti_1port_device = { .driver = { .name = "ti_usb_3410_5052_1", }, .description = "TI USB 3410 1 port adapter", .id_table = ti_id_table_3410, .num_ports = 1, .num_bulk_out = 1, .attach = ti_startup, .release = ti_release, .port_probe = ti_port_probe, .port_remove = ti_port_remove, .open = ti_open, .close = ti_close, .write = ti_write, .write_room = ti_write_room, .chars_in_buffer = ti_chars_in_buffer, .tx_empty = ti_tx_empty, .throttle = ti_throttle, .unthrottle = ti_unthrottle, .get_serial = ti_get_serial_info, .set_termios = ti_set_termios, .tiocmget = ti_tiocmget, .tiocmset = ti_tiocmset, .tiocmiwait = usb_serial_generic_tiocmiwait, .get_icount = usb_serial_generic_get_icount, .break_ctl = ti_break, .read_int_callback = ti_interrupt_callback, .read_bulk_callback = ti_bulk_in_callback, .write_bulk_callback = ti_bulk_out_callback, }; static struct usb_serial_driver ti_2port_device = { .driver = { .name = "ti_usb_3410_5052_2", }, .description = "TI USB 5052 2 port adapter", .id_table = ti_id_table_5052, .num_ports = 2, .num_bulk_out = 1, .attach = ti_startup, .release = ti_release, .port_probe = ti_port_probe, .port_remove = ti_port_remove, .open = ti_open, .close = ti_close, .write = ti_write, .write_room = ti_write_room, .chars_in_buffer = ti_chars_in_buffer, .tx_empty = ti_tx_empty, .throttle = ti_throttle, .unthrottle = ti_unthrottle, .get_serial = ti_get_serial_info, .set_termios = ti_set_termios, .tiocmget = ti_tiocmget, .tiocmset = ti_tiocmset, .tiocmiwait = usb_serial_generic_tiocmiwait, .get_icount = usb_serial_generic_get_icount, .break_ctl = ti_break, .read_int_callback = ti_interrupt_callback, .read_bulk_callback = ti_bulk_in_callback, .write_bulk_callback = ti_bulk_out_callback, }; static struct usb_serial_driver * const serial_drivers[] = { &ti_1port_device, &ti_2port_device, NULL }; MODULE_AUTHOR(TI_DRIVER_AUTHOR); MODULE_DESCRIPTION(TI_DRIVER_DESC); MODULE_LICENSE("GPL"); MODULE_FIRMWARE("ti_3410.fw"); MODULE_FIRMWARE("ti_5052.fw"); MODULE_FIRMWARE("mts_cdma.fw"); MODULE_FIRMWARE("mts_gsm.fw"); MODULE_FIRMWARE("mts_edge.fw"); MODULE_FIRMWARE("mts_mt9234mu.fw"); MODULE_FIRMWARE("mts_mt9234zba.fw"); MODULE_FIRMWARE("moxa/moxa-1110.fw"); MODULE_FIRMWARE("moxa/moxa-1130.fw"); MODULE_FIRMWARE("moxa/moxa-1131.fw"); MODULE_FIRMWARE("moxa/moxa-1150.fw"); MODULE_FIRMWARE("moxa/moxa-1151.fw"); MODULE_DEVICE_TABLE(usb, ti_id_table_combined); module_usb_serial_driver(serial_drivers, ti_id_table_combined); static int ti_startup(struct usb_serial *serial) { struct ti_device *tdev; struct usb_device *dev = serial->dev; struct usb_host_interface *cur_altsetting; int num_endpoints; u16 vid, pid; int status; dev_dbg(&dev->dev, "%s - product 0x%4X, num configurations %d, configuration value %d\n", __func__, le16_to_cpu(dev->descriptor.idProduct), dev->descriptor.bNumConfigurations, dev->actconfig->desc.bConfigurationValue); tdev = kzalloc(sizeof(struct ti_device), GFP_KERNEL); if (!tdev) return -ENOMEM; mutex_init(&tdev->td_open_close_lock); tdev->td_serial = serial; usb_set_serial_data(serial, tdev); /* determine device type */ if (serial->type == &ti_1port_device) tdev->td_is_3410 = 1; dev_dbg(&dev->dev, "%s - device type is %s\n", __func__, tdev->td_is_3410 ? "3410" : "5052"); vid = le16_to_cpu(dev->descriptor.idVendor); pid = le16_to_cpu(dev->descriptor.idProduct); if (vid == MXU1_VENDOR_ID) { switch (pid) { case MXU1_1130_PRODUCT_ID: case MXU1_1131_PRODUCT_ID: tdev->td_rs485_only = true; break; } } cur_altsetting = serial->interface->cur_altsetting; num_endpoints = cur_altsetting->desc.bNumEndpoints; /* if we have only 1 configuration and 1 endpoint, download firmware */ if (dev->descriptor.bNumConfigurations == 1 && num_endpoints == 1) { status = ti_download_firmware(tdev); if (status != 0) goto free_tdev; /* 3410 must be reset, 5052 resets itself */ if (tdev->td_is_3410) { msleep_interruptible(100); usb_reset_device(dev); } status = -ENODEV; goto free_tdev; } /* the second configuration must be set */ if (dev->actconfig->desc.bConfigurationValue == TI_BOOT_CONFIG) { status = usb_driver_set_configuration(dev, TI_ACTIVE_CONFIG); status = status ? status : -ENODEV; goto free_tdev; } if (serial->num_bulk_in < serial->num_ports || serial->num_bulk_out < serial->num_ports) { dev_err(&serial->interface->dev, "missing endpoints\n"); status = -ENODEV; goto free_tdev; } return 0; free_tdev: kfree(tdev); usb_set_serial_data(serial, NULL); return status; } static void ti_release(struct usb_serial *serial) { struct ti_device *tdev = usb_get_serial_data(serial); kfree(tdev); } static int ti_port_probe(struct usb_serial_port *port) { struct ti_port *tport; tport = kzalloc(sizeof(*tport), GFP_KERNEL); if (!tport) return -ENOMEM; spin_lock_init(&tport->tp_lock); if (port == port->serial->port[0]) tport->tp_uart_base_addr = TI_UART1_BASE_ADDR; else tport->tp_uart_base_addr = TI_UART2_BASE_ADDR; tport->tp_port = port; tport->tp_tdev = usb_get_serial_data(port->serial); if (tport->tp_tdev->td_rs485_only) tport->tp_uart_mode = TI_UART_485_RECEIVER_DISABLED; else tport->tp_uart_mode = TI_UART_232; usb_set_serial_port_data(port, tport); /* * The TUSB5052 LSR does not tell when the transmitter shift register * has emptied so add a one-character drain delay. */ if (!tport->tp_tdev->td_is_3410) port->port.drain_delay = 1; return 0; } static void ti_port_remove(struct usb_serial_port *port) { struct ti_port *tport; tport = usb_get_serial_port_data(port); kfree(tport); } static int ti_open(struct tty_struct *tty, struct usb_serial_port *port) { struct ti_port *tport = usb_get_serial_port_data(port); struct ti_device *tdev; struct usb_device *dev; struct urb *urb; int status; u16 open_settings; open_settings = (TI_PIPE_MODE_CONTINUOUS | TI_PIPE_TIMEOUT_ENABLE | (TI_TRANSFER_TIMEOUT << 2)); dev = port->serial->dev; tdev = tport->tp_tdev; /* only one open on any port on a device at a time */ if (mutex_lock_interruptible(&tdev->td_open_close_lock)) return -ERESTARTSYS; tport->tp_msr = 0; tport->tp_shadow_mcr |= (TI_MCR_RTS | TI_MCR_DTR); /* start interrupt urb the first time a port is opened on this device */ if (tdev->td_open_port_count == 0) { dev_dbg(&port->dev, "%s - start interrupt in urb\n", __func__); urb = tdev->td_serial->port[0]->interrupt_in_urb; if (!urb) { dev_err(&port->dev, "%s - no interrupt urb\n", __func__); status = -EINVAL; goto release_lock; } urb->context = tdev; status = usb_submit_urb(urb, GFP_KERNEL); if (status) { dev_err(&port->dev, "%s - submit interrupt urb failed, %d\n", __func__, status); goto release_lock; } } if (tty) ti_set_termios(tty, port, &tty->termios); status = ti_port_cmd_out(port, TI_OPEN_PORT, open_settings, NULL, 0); if (status) { dev_err(&port->dev, "%s - cannot send open command, %d\n", __func__, status); goto unlink_int_urb; } status = ti_port_cmd_out(port, TI_START_PORT, 0, NULL, 0); if (status) { dev_err(&port->dev, "%s - cannot send start command, %d\n", __func__, status); goto unlink_int_urb; } status = ti_port_cmd_out(port, TI_PURGE_PORT, TI_PURGE_INPUT, NULL, 0); if (status) { dev_err(&port->dev, "%s - cannot clear input buffers, %d\n", __func__, status); goto unlink_int_urb; } status = ti_port_cmd_out(port, TI_PURGE_PORT, TI_PURGE_OUTPUT, NULL, 0); if (status) { dev_err(&port->dev, "%s - cannot clear output buffers, %d\n", __func__, status); goto unlink_int_urb; } /* reset the data toggle on the bulk endpoints to work around bug in * host controllers where things get out of sync some times */ usb_clear_halt(dev, port->write_urb->pipe); usb_clear_halt(dev, port->read_urb->pipe); if (tty) ti_set_termios(tty, port, &tty->termios); status = ti_port_cmd_out(port, TI_OPEN_PORT, open_settings, NULL, 0); if (status) { dev_err(&port->dev, "%s - cannot send open command (2), %d\n", __func__, status); goto unlink_int_urb; } status = ti_port_cmd_out(port, TI_START_PORT, 0, NULL, 0); if (status) { dev_err(&port->dev, "%s - cannot send start command (2), %d\n", __func__, status); goto unlink_int_urb; } /* start read urb */ urb = port->read_urb; tport->tp_read_urb_state = TI_READ_URB_RUNNING; urb->context = tport; status = usb_submit_urb(urb, GFP_KERNEL); if (status) { dev_err(&port->dev, "%s - submit read urb failed, %d\n", __func__, status); goto unlink_int_urb; } tport->tp_is_open = 1; ++tdev->td_open_port_count; goto release_lock; unlink_int_urb: if (tdev->td_open_port_count == 0) usb_kill_urb(port->serial->port[0]->interrupt_in_urb); release_lock: mutex_unlock(&tdev->td_open_close_lock); return status; } static void ti_close(struct usb_serial_port *port) { struct ti_device *tdev; struct ti_port *tport; int status; unsigned long flags; tdev = usb_get_serial_data(port->serial); tport = usb_get_serial_port_data(port); tport->tp_is_open = 0; usb_kill_urb(port->read_urb); usb_kill_urb(port->write_urb); tport->tp_write_urb_in_use = 0; spin_lock_irqsave(&tport->tp_lock, flags); kfifo_reset_out(&port->write_fifo); spin_unlock_irqrestore(&tport->tp_lock, flags); status = ti_port_cmd_out(port, TI_CLOSE_PORT, 0, NULL, 0); if (status) dev_err(&port->dev, "%s - cannot send close port command, %d\n" , __func__, status); mutex_lock(&tdev->td_open_close_lock); --tdev->td_open_port_count; if (tdev->td_open_port_count == 0) { /* last port is closed, shut down interrupt urb */ usb_kill_urb(port->serial->port[0]->interrupt_in_urb); } mutex_unlock(&tdev->td_open_close_lock); } static int ti_write(struct tty_struct *tty, struct usb_serial_port *port, const unsigned char *data, int count) { struct ti_port *tport = usb_get_serial_port_data(port); if (count == 0) { return 0; } if (!tport->tp_is_open) return -ENODEV; count = kfifo_in_locked(&port->write_fifo, data, count, &tport->tp_lock); ti_send(tport); return count; } static unsigned int ti_write_room(struct tty_struct *tty) { struct usb_serial_port *port = tty->driver_data; struct ti_port *tport = usb_get_serial_port_data(port); unsigned int room; unsigned long flags; spin_lock_irqsave(&tport->tp_lock, flags); room = kfifo_avail(&port->write_fifo); spin_unlock_irqrestore(&tport->tp_lock, flags); dev_dbg(&port->dev, "%s - returns %u\n", __func__, room); return room; } static unsigned int ti_chars_in_buffer(struct tty_struct *tty) { struct usb_serial_port *port = tty->driver_data; struct ti_port *tport = usb_get_serial_port_data(port); unsigned int chars; unsigned long flags; spin_lock_irqsave(&tport->tp_lock, flags); chars = kfifo_len(&port->write_fifo); spin_unlock_irqrestore(&tport->tp_lock, flags); dev_dbg(&port->dev, "%s - returns %u\n", __func__, chars); return chars; } static bool ti_tx_empty(struct usb_serial_port *port) { struct ti_port *tport = usb_get_serial_port_data(port); u8 lsr, mask; int ret; /* * TUSB5052 does not have the TEMT bit to tell if the shift register * is empty. */ if (tport->tp_tdev->td_is_3410) mask = TI_LSR_TX_EMPTY_BOTH; else mask = TI_LSR_TX_EMPTY; ret = ti_get_lsr(tport, &lsr); if (!ret && !(lsr & mask)) return false; return true; } static void ti_throttle(struct tty_struct *tty) { struct usb_serial_port *port = tty->driver_data; struct ti_port *tport = usb_get_serial_port_data(port); if (I_IXOFF(tty) || C_CRTSCTS(tty)) ti_stop_read(tport, tty); } static void ti_unthrottle(struct tty_struct *tty) { struct usb_serial_port *port = tty->driver_data; struct ti_port *tport = usb_get_serial_port_data(port); int status; if (I_IXOFF(tty) || C_CRTSCTS(tty)) { status = ti_restart_read(tport, tty); if (status) dev_err(&port->dev, "%s - cannot restart read, %d\n", __func__, status); } } static void ti_set_termios(struct tty_struct *tty, struct usb_serial_port *port, const struct ktermios *old_termios) { struct ti_port *tport = usb_get_serial_port_data(port); struct ti_uart_config *config; int baud; int status; unsigned int mcr; u16 wbaudrate; u16 wflags = 0; config = kmalloc(sizeof(*config), GFP_KERNEL); if (!config) return; /* these flags must be set */ wflags |= TI_UART_ENABLE_MS_INTS; wflags |= TI_UART_ENABLE_AUTO_START_DMA; config->bUartMode = tport->tp_uart_mode; switch (C_CSIZE(tty)) { case CS5: config->bDataBits = TI_UART_5_DATA_BITS; break; case CS6: config->bDataBits = TI_UART_6_DATA_BITS; break; case CS7: config->bDataBits = TI_UART_7_DATA_BITS; break; default: case CS8: config->bDataBits = TI_UART_8_DATA_BITS; break; } /* CMSPAR isn't supported by this driver */ tty->termios.c_cflag &= ~CMSPAR; if (C_PARENB(tty)) { if (C_PARODD(tty)) { wflags |= TI_UART_ENABLE_PARITY_CHECKING; config->bParity = TI_UART_ODD_PARITY; } else { wflags |= TI_UART_ENABLE_PARITY_CHECKING; config->bParity = TI_UART_EVEN_PARITY; } } else { wflags &= ~TI_UART_ENABLE_PARITY_CHECKING; config->bParity = TI_UART_NO_PARITY; } if (C_CSTOPB(tty)) config->bStopBits = TI_UART_2_STOP_BITS; else config->bStopBits = TI_UART_1_STOP_BITS; if (C_CRTSCTS(tty)) { /* RTS flow control must be off to drop RTS for baud rate B0 */ if ((C_BAUD(tty)) != B0) wflags |= TI_UART_ENABLE_RTS_IN; wflags |= TI_UART_ENABLE_CTS_OUT; } else { ti_restart_read(tport, tty); } if (I_IXOFF(tty) || I_IXON(tty)) { config->cXon = START_CHAR(tty); config->cXoff = STOP_CHAR(tty); if (I_IXOFF(tty)) wflags |= TI_UART_ENABLE_X_IN; else ti_restart_read(tport, tty); if (I_IXON(tty)) wflags |= TI_UART_ENABLE_X_OUT; } baud = tty_get_baud_rate(tty); if (!baud) baud = 9600; if (tport->tp_tdev->td_is_3410) wbaudrate = (923077 + baud/2) / baud; else wbaudrate = (461538 + baud/2) / baud; /* FIXME: Should calculate resulting baud here and report it back */ if ((C_BAUD(tty)) != B0) tty_encode_baud_rate(tty, baud, baud); dev_dbg(&port->dev, "%s - BaudRate=%d, wBaudRate=%d, wFlags=0x%04X, bDataBits=%d, bParity=%d, bStopBits=%d, cXon=%d, cXoff=%d, bUartMode=%d\n", __func__, baud, wbaudrate, wflags, config->bDataBits, config->bParity, config->bStopBits, config->cXon, config->cXoff, config->bUartMode); config->wBaudRate = cpu_to_be16(wbaudrate); config->wFlags = cpu_to_be16(wflags); status = ti_port_cmd_out(port, TI_SET_CONFIG, 0, config, sizeof(*config)); if (status) dev_err(&port->dev, "%s - cannot set config on port %d, %d\n", __func__, port->port_number, status); /* SET_CONFIG asserts RTS and DTR, reset them correctly */ mcr = tport->tp_shadow_mcr; /* if baud rate is B0, clear RTS and DTR */ if (C_BAUD(tty) == B0) mcr &= ~(TI_MCR_DTR | TI_MCR_RTS); status = ti_set_mcr(tport, mcr); if (status) dev_err(&port->dev, "%s - cannot set modem control on port %d, %d\n", __func__, port->port_number, status); kfree(config); } static int ti_tiocmget(struct tty_struct *tty) { struct usb_serial_port *port = tty->driver_data; struct ti_port *tport = usb_get_serial_port_data(port); unsigned int result; unsigned int msr; unsigned int mcr; unsigned long flags; spin_lock_irqsave(&tport->tp_lock, flags); msr = tport->tp_msr; mcr = tport->tp_shadow_mcr; spin_unlock_irqrestore(&tport->tp_lock, flags); result = ((mcr & TI_MCR_DTR) ? TIOCM_DTR : 0) | ((mcr & TI_MCR_RTS) ? TIOCM_RTS : 0) | ((mcr & TI_MCR_LOOP) ? TIOCM_LOOP : 0) | ((msr & TI_MSR_CTS) ? TIOCM_CTS : 0) | ((msr & TI_MSR_CD) ? TIOCM_CAR : 0) | ((msr & TI_MSR_RI) ? TIOCM_RI : 0) | ((msr & TI_MSR_DSR) ? TIOCM_DSR : 0); dev_dbg(&port->dev, "%s - 0x%04X\n", __func__, result); return result; } static int ti_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear) { struct usb_serial_port *port = tty->driver_data; struct ti_port *tport = usb_get_serial_port_data(port); unsigned int mcr; unsigned long flags; spin_lock_irqsave(&tport->tp_lock, flags); mcr = tport->tp_shadow_mcr; if (set & TIOCM_RTS) mcr |= TI_MCR_RTS; if (set & TIOCM_DTR) mcr |= TI_MCR_DTR; if (set & TIOCM_LOOP) mcr |= TI_MCR_LOOP; if (clear & TIOCM_RTS) mcr &= ~TI_MCR_RTS; if (clear & TIOCM_DTR) mcr &= ~TI_MCR_DTR; if (clear & TIOCM_LOOP) mcr &= ~TI_MCR_LOOP; spin_unlock_irqrestore(&tport->tp_lock, flags); return ti_set_mcr(tport, mcr); } static int ti_break(struct tty_struct *tty, int break_state) { struct usb_serial_port *port = tty->driver_data; struct ti_port *tport = usb_get_serial_port_data(port); int status; dev_dbg(&port->dev, "%s - state = %d\n", __func__, break_state); status = ti_write_byte(port, tport->tp_tdev, tport->tp_uart_base_addr + TI_UART_OFFSET_LCR, TI_LCR_BREAK, break_state == -1 ? TI_LCR_BREAK : 0); if (status) { dev_dbg(&port->dev, "%s - error setting break, %d\n", __func__, status); return status; } return 0; } static int ti_get_port_from_code(unsigned char code) { return (code >> 6) & 0x01; } static int ti_get_func_from_code(unsigned char code) { return code & 0x0f; } static void ti_interrupt_callback(struct urb *urb) { struct ti_device *tdev = urb->context; struct usb_serial_port *port; struct usb_serial *serial = tdev->td_serial; struct ti_port *tport; struct device *dev = &urb->dev->dev; unsigned char *data = urb->transfer_buffer; int length = urb->actual_length; int port_number; int function; int status = urb->status; int retval; u8 msr; switch (status) { case 0: break; case -ECONNRESET: case -ENOENT: case -ESHUTDOWN: dev_dbg(dev, "%s - urb shutting down, %d\n", __func__, status); return; default: dev_err(dev, "%s - nonzero urb status, %d\n", __func__, status); goto exit; } if (length != 2) { dev_dbg(dev, "%s - bad packet size, %d\n", __func__, length); goto exit; } if (data[0] == TI_CODE_HARDWARE_ERROR) { dev_err(dev, "%s - hardware error, %d\n", __func__, data[1]); goto exit; } port_number = ti_get_port_from_code(data[0]); function = ti_get_func_from_code(data[0]); dev_dbg(dev, "%s - port_number %d, function %d, data 0x%02X\n", __func__, port_number, function, data[1]); if (port_number >= serial->num_ports) { dev_err(dev, "%s - bad port number, %d\n", __func__, port_number); goto exit; } port = serial->port[port_number]; tport = usb_get_serial_port_data(port); if (!tport) goto exit; switch (function) { case TI_CODE_DATA_ERROR: dev_err(dev, "%s - DATA ERROR, port %d, data 0x%02X\n", __func__, port_number, data[1]); break; case TI_CODE_MODEM_STATUS: msr = data[1]; dev_dbg(dev, "%s - port %d, msr 0x%02X\n", __func__, port_number, msr); ti_handle_new_msr(tport, msr); break; default: dev_err(dev, "%s - unknown interrupt code, 0x%02X\n", __func__, data[1]); break; } exit: retval = usb_submit_urb(urb, GFP_ATOMIC); if (retval) dev_err(dev, "%s - resubmit interrupt urb failed, %d\n", __func__, retval); } static void ti_bulk_in_callback(struct urb *urb) { struct ti_port *tport = urb->context; struct usb_serial_port *port = tport->tp_port; struct device *dev = &urb->dev->dev; int status = urb->status; unsigned long flags; int retval = 0; switch (status) { case 0: break; case -ECONNRESET: case -ENOENT: case -ESHUTDOWN: dev_dbg(dev, "%s - urb shutting down, %d\n", __func__, status); return; default: dev_err(dev, "%s - nonzero urb status, %d\n", __func__, status); } if (status == -EPIPE) goto exit; if (status) { dev_err(dev, "%s - stopping read!\n", __func__); return; } if (urb->actual_length) { usb_serial_debug_data(dev, __func__, urb->actual_length, urb->transfer_buffer); if (!tport->tp_is_open) dev_dbg(dev, "%s - port closed, dropping data\n", __func__); else ti_recv(port, urb->transfer_buffer, urb->actual_length); spin_lock_irqsave(&tport->tp_lock, flags); port->icount.rx += urb->actual_length; spin_unlock_irqrestore(&tport->tp_lock, flags); } exit: /* continue to read unless stopping */ spin_lock_irqsave(&tport->tp_lock, flags); if (tport->tp_read_urb_state == TI_READ_URB_RUNNING) retval = usb_submit_urb(urb, GFP_ATOMIC); else if (tport->tp_read_urb_state == TI_READ_URB_STOPPING) tport->tp_read_urb_state = TI_READ_URB_STOPPED; spin_unlock_irqrestore(&tport->tp_lock, flags); if (retval) dev_err(dev, "%s - resubmit read urb failed, %d\n", __func__, retval); } static void ti_bulk_out_callback(struct urb *urb) { struct ti_port *tport = urb->context; struct usb_serial_port *port = tport->tp_port; int status = urb->status; tport->tp_write_urb_in_use = 0; switch (status) { case 0: break; case -ECONNRESET: case -ENOENT: case -ESHUTDOWN: dev_dbg(&port->dev, "%s - urb shutting down, %d\n", __func__, status); return; default: dev_err_console(port, "%s - nonzero urb status, %d\n", __func__, status); } /* send any buffered data */ ti_send(tport); } static void ti_recv(struct usb_serial_port *port, unsigned char *data, int length) { int cnt; do { cnt = tty_insert_flip_string(&port->port, data, length); if (cnt < length) { dev_err(&port->dev, "%s - dropping data, %d bytes lost\n", __func__, length - cnt); if (cnt == 0) break; } tty_flip_buffer_push(&port->port); data += cnt; length -= cnt; } while (length > 0); } static void ti_send(struct ti_port *tport) { int count, result; struct usb_serial_port *port = tport->tp_port; unsigned long flags; spin_lock_irqsave(&tport->tp_lock, flags); if (tport->tp_write_urb_in_use) goto unlock; count = kfifo_out(&port->write_fifo, port->write_urb->transfer_buffer, port->bulk_out_size); if (count == 0) goto unlock; tport->tp_write_urb_in_use = 1; spin_unlock_irqrestore(&tport->tp_lock, flags); usb_serial_debug_data(&port->dev, __func__, count, port->write_urb->transfer_buffer); usb_fill_bulk_urb(port->write_urb, port->serial->dev, usb_sndbulkpipe(port->serial->dev, port->bulk_out_endpointAddress), port->write_urb->transfer_buffer, count, ti_bulk_out_callback, tport); result = usb_submit_urb(port->write_urb, GFP_ATOMIC); if (result) { dev_err_console(port, "%s - submit write urb failed, %d\n", __func__, result); tport->tp_write_urb_in_use = 0; /* TODO: reschedule ti_send */ } else { spin_lock_irqsave(&tport->tp_lock, flags); port->icount.tx += count; spin_unlock_irqrestore(&tport->tp_lock, flags); } /* more room in the buffer for new writes, wakeup */ tty_port_tty_wakeup(&port->port); return; unlock: spin_unlock_irqrestore(&tport->tp_lock, flags); return; } static int ti_set_mcr(struct ti_port *tport, unsigned int mcr) { unsigned long flags; int status; status = ti_write_byte(tport->tp_port, tport->tp_tdev, tport->tp_uart_base_addr + TI_UART_OFFSET_MCR, TI_MCR_RTS | TI_MCR_DTR | TI_MCR_LOOP, mcr); spin_lock_irqsave(&tport->tp_lock, flags); if (!status) tport->tp_shadow_mcr = mcr; spin_unlock_irqrestore(&tport->tp_lock, flags); return status; } static int ti_get_lsr(struct ti_port *tport, u8 *lsr) { int size, status; struct usb_serial_port *port = tport->tp_port; struct ti_port_status *data; size = sizeof(struct ti_port_status); data = kmalloc(size, GFP_KERNEL); if (!data) return -ENOMEM; status = ti_port_cmd_in(port, TI_GET_PORT_STATUS, 0, data, size); if (status) { dev_err(&port->dev, "%s - get port status command failed, %d\n", __func__, status); goto free_data; } dev_dbg(&port->dev, "%s - lsr 0x%02X\n", __func__, data->bLSR); *lsr = data->bLSR; free_data: kfree(data); return status; } static void ti_get_serial_info(struct tty_struct *tty, struct serial_struct *ss) { struct usb_serial_port *port = tty->driver_data; struct ti_port *tport = usb_get_serial_port_data(port); ss->baud_base = tport->tp_tdev->td_is_3410 ? 921600 : 460800; } static void ti_handle_new_msr(struct ti_port *tport, u8 msr) { struct async_icount *icount; struct tty_struct *tty; unsigned long flags; dev_dbg(&tport->tp_port->dev, "%s - msr 0x%02X\n", __func__, msr); if (msr & TI_MSR_DELTA_MASK) { spin_lock_irqsave(&tport->tp_lock, flags); icount = &tport->tp_port->icount; if (msr & TI_MSR_DELTA_CTS) icount->cts++; if (msr & TI_MSR_DELTA_DSR) icount->dsr++; if (msr & TI_MSR_DELTA_CD) icount->dcd++; if (msr & TI_MSR_DELTA_RI) icount->rng++; wake_up_interruptible(&tport->tp_port->port.delta_msr_wait); spin_unlock_irqrestore(&tport->tp_lock, flags); } tport->tp_msr = msr & TI_MSR_MASK; /* handle CTS flow control */ tty = tty_port_tty_get(&tport->tp_port->port); if (tty && C_CRTSCTS(tty)) { if (msr & TI_MSR_CTS) tty_wakeup(tty); } tty_kref_put(tty); } static void ti_stop_read(struct ti_port *tport, struct tty_struct *tty) { unsigned long flags; spin_lock_irqsave(&tport->tp_lock, flags); if (tport->tp_read_urb_state == TI_READ_URB_RUNNING) tport->tp_read_urb_state = TI_READ_URB_STOPPING; spin_unlock_irqrestore(&tport->tp_lock, flags); } static int ti_restart_read(struct ti_port *tport, struct tty_struct *tty) { struct urb *urb; int status = 0; unsigned long flags; spin_lock_irqsave(&tport->tp_lock, flags); if (tport->tp_read_urb_state == TI_READ_URB_STOPPED) { tport->tp_read_urb_state = TI_READ_URB_RUNNING; urb = tport->tp_port->read_urb; spin_unlock_irqrestore(&tport->tp_lock, flags); urb->context = tport; status = usb_submit_urb(urb, GFP_KERNEL); } else { tport->tp_read_urb_state = TI_READ_URB_RUNNING; spin_unlock_irqrestore(&tport->tp_lock, flags); } return status; } static int ti_command_out_sync(struct usb_device *udev, u8 command, u16 moduleid, u16 value, void *data, int size) { int status; status = usb_control_msg(udev, usb_sndctrlpipe(udev, 0), command, USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT, value, moduleid, data, size, 1000); if (status < 0) return status; return 0; } static int ti_command_in_sync(struct usb_device *udev, u8 command, u16 moduleid, u16 value, void *data, int size) { int status; status = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0), command, USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_IN, value, moduleid, data, size, 1000); if (status == size) status = 0; else if (status >= 0) status = -ECOMM; return status; } static int ti_port_cmd_out(struct usb_serial_port *port, u8 command, u16 value, void *data, int size) { return ti_command_out_sync(port->serial->dev, command, TI_UART1_PORT + port->port_number, value, data, size); } static int ti_port_cmd_in(struct usb_serial_port *port, u8 command, u16 value, void *data, int size) { return ti_command_in_sync(port->serial->dev, command, TI_UART1_PORT + port->port_number, value, data, size); } static int ti_write_byte(struct usb_serial_port *port, struct ti_device *tdev, unsigned long addr, u8 mask, u8 byte) { int status; unsigned int size; struct ti_write_data_bytes *data; dev_dbg(&port->dev, "%s - addr 0x%08lX, mask 0x%02X, byte 0x%02X\n", __func__, addr, mask, byte); size = sizeof(struct ti_write_data_bytes) + 2; data = kmalloc(size, GFP_KERNEL); if (!data) return -ENOMEM; data->bAddrType = TI_RW_DATA_ADDR_XDATA; data->bDataType = TI_RW_DATA_BYTE; data->bDataCounter = 1; data->wBaseAddrHi = cpu_to_be16(addr>>16); data->wBaseAddrLo = cpu_to_be16(addr); data->bData[0] = mask; data->bData[1] = byte; status = ti_command_out_sync(port->serial->dev, TI_WRITE_DATA, TI_RAM_PORT, 0, data, size); if (status < 0) dev_err(&port->dev, "%s - failed, %d\n", __func__, status); kfree(data); return status; } static int ti_do_download(struct usb_device *dev, int pipe, u8 *buffer, int size) { int pos; u8 cs = 0; int done; struct ti_firmware_header *header; int status = 0; int len; for (pos = sizeof(struct ti_firmware_header); pos < size; pos++) cs = (u8)(cs + buffer[pos]); header = (struct ti_firmware_header *)buffer; header->wLength = cpu_to_le16(size - sizeof(*header)); header->bCheckSum = cs; dev_dbg(&dev->dev, "%s - downloading firmware\n", __func__); for (pos = 0; pos < size; pos += done) { len = min(size - pos, TI_DOWNLOAD_MAX_PACKET_SIZE); status = usb_bulk_msg(dev, pipe, buffer + pos, len, &done, 1000); if (status) break; } return status; } static int ti_download_firmware(struct ti_device *tdev) { int status; int buffer_size; u8 *buffer; struct usb_device *dev = tdev->td_serial->dev; unsigned int pipe = usb_sndbulkpipe(dev, tdev->td_serial->port[0]->bulk_out_endpointAddress); const struct firmware *fw_p; char buf[32]; if (le16_to_cpu(dev->descriptor.idVendor) == MXU1_VENDOR_ID) { snprintf(buf, sizeof(buf), "moxa/moxa-%04x.fw", le16_to_cpu(dev->descriptor.idProduct)); status = request_firmware(&fw_p, buf, &dev->dev); goto check_firmware; } /* try ID specific firmware first, then try generic firmware */ sprintf(buf, "ti_usb-v%04x-p%04x.fw", le16_to_cpu(dev->descriptor.idVendor), le16_to_cpu(dev->descriptor.idProduct)); status = request_firmware(&fw_p, buf, &dev->dev); if (status != 0) { buf[0] = '\0'; if (le16_to_cpu(dev->descriptor.idVendor) == MTS_VENDOR_ID) { switch (le16_to_cpu(dev->descriptor.idProduct)) { case MTS_CDMA_PRODUCT_ID: strcpy(buf, "mts_cdma.fw"); break; case MTS_GSM_PRODUCT_ID: strcpy(buf, "mts_gsm.fw"); break; case MTS_EDGE_PRODUCT_ID: strcpy(buf, "mts_edge.fw"); break; case MTS_MT9234MU_PRODUCT_ID: strcpy(buf, "mts_mt9234mu.fw"); break; case MTS_MT9234ZBA_PRODUCT_ID: strcpy(buf, "mts_mt9234zba.fw"); break; case MTS_MT9234ZBAOLD_PRODUCT_ID: strcpy(buf, "mts_mt9234zba.fw"); break; } } if (buf[0] == '\0') { if (tdev->td_is_3410) strcpy(buf, "ti_3410.fw"); else strcpy(buf, "ti_5052.fw"); } status = request_firmware(&fw_p, buf, &dev->dev); } check_firmware: if (status) { dev_err(&dev->dev, "%s - firmware not found\n", __func__); return -ENOENT; } if (fw_p->size > TI_FIRMWARE_BUF_SIZE) { dev_err(&dev->dev, "%s - firmware too large %zu\n", __func__, fw_p->size); release_firmware(fw_p); return -ENOENT; } buffer_size = TI_FIRMWARE_BUF_SIZE + sizeof(struct ti_firmware_header); buffer = kmalloc(buffer_size, GFP_KERNEL); if (buffer) { memcpy(buffer, fw_p->data, fw_p->size); memset(buffer + fw_p->size, 0xff, buffer_size - fw_p->size); status = ti_do_download(dev, pipe, buffer, fw_p->size); kfree(buffer); } else { status = -ENOMEM; } release_firmware(fw_p); if (status) { dev_err(&dev->dev, "%s - error downloading firmware, %d\n", __func__, status); return status; } dev_dbg(&dev->dev, "%s - download successful\n", __func__); return 0; } |
| 16 16 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848 6849 6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7017 7018 7019 7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480 7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497 7498 7499 7500 7501 7502 7503 7504 7505 7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673 7674 7675 7676 7677 7678 7679 7680 7681 7682 7683 7684 7685 7686 7687 7688 7689 7690 7691 7692 7693 7694 7695 7696 7697 7698 7699 7700 7701 7702 7703 7704 7705 7706 7707 7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735 7736 7737 7738 7739 7740 7741 7742 7743 7744 7745 7746 7747 7748 7749 7750 7751 7752 7753 7754 7755 7756 7757 7758 7759 7760 7761 7762 7763 7764 7765 7766 7767 7768 7769 7770 7771 7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885 7886 7887 7888 7889 7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963 7964 7965 7966 7967 7968 7969 7970 7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983 7984 7985 7986 7987 7988 7989 7990 7991 7992 7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009 8010 8011 8012 8013 8014 8015 8016 8017 8018 8019 8020 8021 8022 8023 8024 8025 8026 8027 8028 8029 8030 8031 8032 8033 8034 8035 8036 8037 8038 8039 8040 8041 8042 8043 8044 8045 8046 8047 8048 8049 8050 8051 8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108 8109 8110 8111 8112 8113 8114 8115 8116 8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139 8140 8141 8142 8143 8144 8145 8146 8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160 8161 8162 8163 8164 8165 8166 8167 8168 8169 8170 8171 8172 8173 8174 8175 8176 8177 8178 8179 8180 8181 8182 8183 8184 8185 8186 8187 8188 8189 8190 8191 8192 8193 8194 8195 8196 8197 8198 8199 8200 8201 8202 8203 8204 8205 8206 8207 8208 8209 8210 8211 8212 8213 8214 8215 8216 8217 8218 8219 8220 8221 8222 8223 8224 8225 8226 8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237 8238 8239 8240 8241 8242 8243 8244 8245 8246 8247 8248 8249 8250 8251 8252 8253 8254 8255 8256 8257 8258 8259 8260 8261 8262 8263 8264 8265 8266 8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308 8309 8310 8311 8312 8313 8314 8315 8316 8317 8318 8319 8320 8321 8322 8323 8324 8325 8326 8327 8328 8329 8330 8331 8332 8333 8334 8335 8336 8337 8338 8339 8340 8341 8342 8343 8344 8345 8346 8347 8348 8349 8350 8351 8352 8353 8354 8355 8356 8357 8358 8359 8360 8361 8362 8363 8364 8365 8366 8367 8368 8369 8370 8371 8372 8373 8374 8375 8376 8377 8378 8379 8380 8381 8382 8383 8384 8385 8386 8387 8388 8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399 8400 8401 8402 8403 8404 8405 8406 8407 8408 8409 8410 8411 8412 8413 8414 8415 8416 8417 8418 8419 8420 8421 8422 8423 8424 8425 8426 8427 8428 8429 8430 8431 8432 8433 8434 8435 8436 8437 8438 8439 8440 8441 8442 8443 8444 8445 8446 8447 8448 8449 8450 8451 8452 8453 8454 8455 8456 8457 8458 8459 8460 8461 8462 8463 8464 8465 8466 8467 8468 8469 8470 8471 8472 8473 8474 8475 8476 8477 8478 8479 8480 8481 8482 8483 8484 8485 8486 8487 8488 8489 8490 8491 8492 8493 8494 8495 8496 8497 8498 8499 8500 8501 8502 8503 8504 8505 8506 8507 8508 8509 8510 8511 8512 8513 8514 8515 8516 8517 8518 8519 8520 8521 8522 8523 8524 8525 8526 8527 8528 8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541 8542 8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556 8557 8558 8559 8560 8561 8562 8563 8564 8565 8566 8567 8568 8569 8570 8571 8572 8573 8574 8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586 8587 8588 8589 8590 8591 8592 8593 8594 8595 8596 8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610 8611 8612 8613 8614 8615 8616 8617 8618 8619 8620 8621 8622 8623 8624 8625 8626 8627 8628 8629 8630 8631 8632 8633 8634 8635 8636 8637 8638 8639 8640 8641 8642 8643 8644 8645 8646 8647 8648 8649 8650 8651 8652 8653 8654 8655 8656 8657 8658 8659 8660 8661 8662 8663 8664 8665 8666 8667 8668 8669 8670 8671 8672 8673 8674 8675 8676 8677 8678 8679 8680 8681 8682 8683 8684 8685 8686 8687 8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711 8712 8713 8714 8715 8716 8717 8718 8719 8720 8721 8722 8723 8724 8725 8726 8727 8728 8729 8730 8731 8732 8733 8734 8735 8736 8737 8738 8739 8740 8741 8742 8743 8744 8745 8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780 8781 8782 8783 8784 8785 8786 8787 8788 8789 8790 8791 8792 8793 8794 8795 8796 8797 8798 8799 8800 8801 8802 8803 8804 8805 8806 8807 8808 8809 8810 8811 8812 8813 8814 8815 8816 8817 8818 8819 8820 8821 8822 8823 8824 8825 8826 8827 8828 8829 8830 8831 8832 8833 8834 8835 8836 8837 8838 8839 8840 8841 8842 8843 8844 8845 8846 8847 8848 8849 8850 8851 8852 8853 8854 8855 8856 8857 8858 8859 8860 8861 8862 8863 8864 8865 8866 8867 8868 8869 8870 8871 8872 8873 8874 8875 8876 8877 8878 8879 8880 8881 8882 8883 8884 8885 8886 8887 8888 8889 8890 8891 8892 8893 8894 8895 8896 8897 8898 8899 8900 8901 8902 8903 8904 8905 8906 8907 8908 8909 8910 8911 8912 8913 8914 8915 8916 8917 8918 8919 8920 8921 8922 8923 8924 8925 8926 8927 8928 8929 8930 8931 8932 8933 8934 8935 8936 8937 8938 8939 8940 8941 8942 8943 8944 8945 8946 8947 8948 8949 8950 8951 8952 8953 8954 8955 8956 8957 8958 8959 8960 8961 8962 8963 8964 8965 8966 8967 8968 8969 8970 8971 8972 8973 8974 8975 8976 8977 8978 8979 8980 8981 8982 8983 8984 8985 8986 8987 8988 8989 8990 8991 8992 8993 8994 8995 8996 8997 8998 8999 9000 9001 9002 9003 9004 9005 9006 9007 9008 9009 9010 9011 9012 9013 9014 9015 9016 9017 9018 9019 9020 9021 9022 9023 9024 9025 9026 9027 9028 9029 9030 9031 9032 9033 9034 9035 9036 9037 9038 9039 9040 9041 9042 9043 9044 9045 9046 9047 9048 9049 9050 9051 9052 9053 9054 9055 9056 9057 9058 9059 9060 9061 9062 9063 9064 9065 9066 9067 9068 9069 9070 9071 9072 9073 9074 9075 9076 9077 9078 9079 9080 9081 9082 9083 9084 9085 9086 9087 9088 9089 9090 9091 9092 9093 9094 9095 9096 9097 9098 9099 9100 9101 9102 9103 9104 9105 9106 9107 9108 9109 9110 9111 9112 9113 9114 9115 9116 9117 9118 9119 9120 9121 9122 9123 9124 9125 9126 9127 9128 9129 9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146 9147 9148 9149 9150 9151 9152 9153 9154 9155 9156 9157 9158 9159 9160 9161 9162 9163 9164 9165 9166 9167 9168 9169 9170 9171 9172 9173 9174 9175 9176 9177 9178 9179 9180 9181 9182 9183 9184 9185 9186 9187 9188 9189 9190 9191 9192 9193 9194 9195 9196 9197 9198 9199 9200 9201 9202 9203 9204 9205 9206 9207 9208 9209 9210 9211 9212 9213 9214 9215 9216 9217 9218 9219 9220 9221 9222 9223 9224 9225 9226 9227 9228 9229 9230 9231 9232 9233 9234 9235 9236 9237 9238 9239 9240 9241 9242 9243 9244 9245 9246 9247 9248 9249 9250 9251 9252 9253 9254 9255 9256 9257 9258 9259 9260 9261 9262 9263 9264 9265 9266 9267 9268 9269 9270 9271 9272 9273 9274 9275 9276 9277 9278 9279 9280 9281 9282 9283 9284 9285 9286 9287 9288 9289 9290 9291 9292 9293 9294 9295 9296 9297 9298 9299 9300 9301 9302 9303 9304 9305 9306 9307 9308 9309 9310 9311 9312 9313 9314 9315 9316 9317 9318 9319 9320 9321 9322 9323 9324 9325 9326 9327 9328 9329 9330 9331 9332 9333 9334 9335 9336 9337 9338 9339 9340 9341 9342 9343 9344 9345 9346 9347 9348 9349 9350 9351 9352 9353 9354 9355 9356 9357 9358 9359 9360 9361 9362 9363 9364 9365 9366 9367 9368 9369 9370 9371 9372 9373 9374 9375 9376 9377 9378 9379 9380 9381 9382 9383 9384 9385 9386 9387 9388 9389 9390 9391 9392 9393 9394 9395 9396 9397 9398 9399 9400 9401 9402 9403 9404 9405 9406 9407 9408 9409 9410 9411 9412 9413 9414 9415 9416 9417 9418 9419 9420 9421 9422 9423 9424 9425 9426 9427 9428 9429 9430 9431 9432 9433 9434 9435 9436 9437 9438 9439 9440 9441 9442 9443 9444 9445 9446 9447 9448 9449 9450 9451 9452 9453 9454 9455 9456 9457 9458 9459 9460 9461 9462 9463 9464 9465 9466 9467 9468 9469 9470 9471 9472 9473 9474 9475 9476 9477 9478 9479 9480 9481 9482 9483 9484 9485 9486 9487 9488 9489 9490 9491 9492 9493 9494 9495 9496 9497 9498 9499 9500 9501 9502 9503 9504 9505 9506 9507 9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526 9527 9528 9529 9530 9531 9532 9533 9534 9535 9536 9537 9538 9539 9540 9541 9542 9543 9544 9545 9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556 9557 9558 9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572 9573 9574 9575 9576 9577 9578 9579 9580 9581 9582 9583 9584 9585 9586 9587 9588 9589 9590 9591 9592 9593 9594 9595 9596 9597 9598 9599 9600 9601 9602 9603 9604 9605 9606 9607 9608 9609 9610 9611 9612 9613 9614 9615 9616 9617 9618 9619 9620 9621 9622 9623 9624 9625 9626 9627 9628 9629 9630 9631 9632 9633 9634 9635 9636 9637 9638 9639 9640 9641 | // SPDX-License-Identifier: ISC /* * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. */ #include <linux/skbuff.h> #include <linux/ctype.h> #include "core.h" #include "htc.h" #include "debug.h" #include "wmi.h" #include "wmi-tlv.h" #include "mac.h" #include "testmode.h" #include "wmi-ops.h" #include "p2p.h" #include "hw.h" #include "hif.h" #include "txrx.h" #define ATH10K_WMI_BARRIER_ECHO_ID 0xBA991E9 #define ATH10K_WMI_BARRIER_TIMEOUT_HZ (3 * HZ) #define ATH10K_WMI_DFS_CONF_TIMEOUT_HZ (HZ / 6) /* MAIN WMI cmd track */ static struct wmi_cmd_map wmi_cmd_map = { .init_cmdid = WMI_INIT_CMDID, .start_scan_cmdid = WMI_START_SCAN_CMDID, .stop_scan_cmdid = WMI_STOP_SCAN_CMDID, .scan_chan_list_cmdid = WMI_SCAN_CHAN_LIST_CMDID, .scan_sch_prio_tbl_cmdid = WMI_SCAN_SCH_PRIO_TBL_CMDID, .scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_regdomain_cmdid = WMI_PDEV_SET_REGDOMAIN_CMDID, .pdev_set_channel_cmdid = WMI_PDEV_SET_CHANNEL_CMDID, .pdev_set_param_cmdid = WMI_PDEV_SET_PARAM_CMDID, .pdev_pktlog_enable_cmdid = WMI_PDEV_PKTLOG_ENABLE_CMDID, .pdev_pktlog_disable_cmdid = WMI_PDEV_PKTLOG_DISABLE_CMDID, .pdev_set_wmm_params_cmdid = WMI_PDEV_SET_WMM_PARAMS_CMDID, .pdev_set_ht_cap_ie_cmdid = WMI_PDEV_SET_HT_CAP_IE_CMDID, .pdev_set_vht_cap_ie_cmdid = WMI_PDEV_SET_VHT_CAP_IE_CMDID, .pdev_set_dscp_tid_map_cmdid = WMI_PDEV_SET_DSCP_TID_MAP_CMDID, .pdev_set_quiet_mode_cmdid = WMI_PDEV_SET_QUIET_MODE_CMDID, .pdev_green_ap_ps_enable_cmdid = WMI_PDEV_GREEN_AP_PS_ENABLE_CMDID, .pdev_get_tpc_config_cmdid = WMI_PDEV_GET_TPC_CONFIG_CMDID, .pdev_set_base_macaddr_cmdid = WMI_PDEV_SET_BASE_MACADDR_CMDID, .vdev_create_cmdid = WMI_VDEV_CREATE_CMDID, .vdev_delete_cmdid = WMI_VDEV_DELETE_CMDID, .vdev_start_request_cmdid = WMI_VDEV_START_REQUEST_CMDID, .vdev_restart_request_cmdid = WMI_VDEV_RESTART_REQUEST_CMDID, .vdev_up_cmdid = WMI_VDEV_UP_CMDID, .vdev_stop_cmdid = WMI_VDEV_STOP_CMDID, .vdev_down_cmdid = WMI_VDEV_DOWN_CMDID, .vdev_set_param_cmdid = WMI_VDEV_SET_PARAM_CMDID, .vdev_install_key_cmdid = WMI_VDEV_INSTALL_KEY_CMDID, .peer_create_cmdid = WMI_PEER_CREATE_CMDID, .peer_delete_cmdid = WMI_PEER_DELETE_CMDID, .peer_flush_tids_cmdid = WMI_PEER_FLUSH_TIDS_CMDID, .peer_set_param_cmdid = WMI_PEER_SET_PARAM_CMDID, .peer_assoc_cmdid = WMI_PEER_ASSOC_CMDID, .peer_add_wds_entry_cmdid = WMI_PEER_ADD_WDS_ENTRY_CMDID, .peer_remove_wds_entry_cmdid = WMI_PEER_REMOVE_WDS_ENTRY_CMDID, .peer_mcast_group_cmdid = WMI_PEER_MCAST_GROUP_CMDID, .bcn_tx_cmdid = WMI_BCN_TX_CMDID, .pdev_send_bcn_cmdid = WMI_PDEV_SEND_BCN_CMDID, .bcn_tmpl_cmdid = WMI_BCN_TMPL_CMDID, .bcn_filter_rx_cmdid = WMI_BCN_FILTER_RX_CMDID, .prb_req_filter_rx_cmdid = WMI_PRB_REQ_FILTER_RX_CMDID, .mgmt_tx_cmdid = WMI_MGMT_TX_CMDID, .prb_tmpl_cmdid = WMI_PRB_TMPL_CMDID, .addba_clear_resp_cmdid = WMI_ADDBA_CLEAR_RESP_CMDID, .addba_send_cmdid = WMI_ADDBA_SEND_CMDID, .addba_status_cmdid = WMI_ADDBA_STATUS_CMDID, .delba_send_cmdid = WMI_DELBA_SEND_CMDID, .addba_set_resp_cmdid = WMI_ADDBA_SET_RESP_CMDID, .send_singleamsdu_cmdid = WMI_SEND_SINGLEAMSDU_CMDID, .sta_powersave_mode_cmdid = WMI_STA_POWERSAVE_MODE_CMDID, .sta_powersave_param_cmdid = WMI_STA_POWERSAVE_PARAM_CMDID, .sta_mimo_ps_mode_cmdid = WMI_STA_MIMO_PS_MODE_CMDID, .pdev_dfs_enable_cmdid = WMI_PDEV_DFS_ENABLE_CMDID, .pdev_dfs_disable_cmdid = WMI_PDEV_DFS_DISABLE_CMDID, .roam_scan_mode = WMI_ROAM_SCAN_MODE, .roam_scan_rssi_threshold = WMI_ROAM_SCAN_RSSI_THRESHOLD, .roam_scan_period = WMI_ROAM_SCAN_PERIOD, .roam_scan_rssi_change_threshold = WMI_ROAM_SCAN_RSSI_CHANGE_THRESHOLD, .roam_ap_profile = WMI_ROAM_AP_PROFILE, .ofl_scan_add_ap_profile = WMI_ROAM_AP_PROFILE, .ofl_scan_remove_ap_profile = WMI_OFL_SCAN_REMOVE_AP_PROFILE, .ofl_scan_period = WMI_OFL_SCAN_PERIOD, .p2p_dev_set_device_info = WMI_P2P_DEV_SET_DEVICE_INFO, .p2p_dev_set_discoverability = WMI_P2P_DEV_SET_DISCOVERABILITY, .p2p_go_set_beacon_ie = WMI_P2P_GO_SET_BEACON_IE, .p2p_go_set_probe_resp_ie = WMI_P2P_GO_SET_PROBE_RESP_IE, .p2p_set_vendor_ie_data_cmdid = WMI_P2P_SET_VENDOR_IE_DATA_CMDID, .ap_ps_peer_param_cmdid = WMI_AP_PS_PEER_PARAM_CMDID, .ap_ps_peer_uapsd_coex_cmdid = WMI_AP_PS_PEER_UAPSD_COEX_CMDID, .peer_rate_retry_sched_cmdid = WMI_PEER_RATE_RETRY_SCHED_CMDID, .wlan_profile_trigger_cmdid = WMI_WLAN_PROFILE_TRIGGER_CMDID, .wlan_profile_set_hist_intvl_cmdid = WMI_WLAN_PROFILE_SET_HIST_INTVL_CMDID, .wlan_profile_get_profile_data_cmdid = WMI_WLAN_PROFILE_GET_PROFILE_DATA_CMDID, .wlan_profile_enable_profile_id_cmdid = WMI_WLAN_PROFILE_ENABLE_PROFILE_ID_CMDID, .wlan_profile_list_profile_id_cmdid = WMI_WLAN_PROFILE_LIST_PROFILE_ID_CMDID, .pdev_suspend_cmdid = WMI_PDEV_SUSPEND_CMDID, .pdev_resume_cmdid = WMI_PDEV_RESUME_CMDID, .add_bcn_filter_cmdid = WMI_ADD_BCN_FILTER_CMDID, .rmv_bcn_filter_cmdid = WMI_RMV_BCN_FILTER_CMDID, .wow_add_wake_pattern_cmdid = WMI_WOW_ADD_WAKE_PATTERN_CMDID, .wow_del_wake_pattern_cmdid = WMI_WOW_DEL_WAKE_PATTERN_CMDID, .wow_enable_disable_wake_event_cmdid = WMI_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID, .wow_enable_cmdid = WMI_WOW_ENABLE_CMDID, .wow_hostwakeup_from_sleep_cmdid = WMI_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID, .rtt_measreq_cmdid = WMI_RTT_MEASREQ_CMDID, .rtt_tsf_cmdid = WMI_RTT_TSF_CMDID, .vdev_spectral_scan_configure_cmdid = WMI_VDEV_SPECTRAL_SCAN_CONFIGURE_CMDID, .vdev_spectral_scan_enable_cmdid = WMI_VDEV_SPECTRAL_SCAN_ENABLE_CMDID, .request_stats_cmdid = WMI_REQUEST_STATS_CMDID, .set_arp_ns_offload_cmdid = WMI_SET_ARP_NS_OFFLOAD_CMDID, .network_list_offload_config_cmdid = WMI_NETWORK_LIST_OFFLOAD_CONFIG_CMDID, .gtk_offload_cmdid = WMI_GTK_OFFLOAD_CMDID, .csa_offload_enable_cmdid = WMI_CSA_OFFLOAD_ENABLE_CMDID, .csa_offload_chanswitch_cmdid = WMI_CSA_OFFLOAD_CHANSWITCH_CMDID, .chatter_set_mode_cmdid = WMI_CHATTER_SET_MODE_CMDID, .peer_tid_addba_cmdid = WMI_PEER_TID_ADDBA_CMDID, .peer_tid_delba_cmdid = WMI_PEER_TID_DELBA_CMDID, .sta_dtim_ps_method_cmdid = WMI_STA_DTIM_PS_METHOD_CMDID, .sta_uapsd_auto_trig_cmdid = WMI_STA_UAPSD_AUTO_TRIG_CMDID, .sta_keepalive_cmd = WMI_STA_KEEPALIVE_CMD, .echo_cmdid = WMI_ECHO_CMDID, .pdev_utf_cmdid = WMI_PDEV_UTF_CMDID, .dbglog_cfg_cmdid = WMI_DBGLOG_CFG_CMDID, .pdev_qvit_cmdid = WMI_PDEV_QVIT_CMDID, .pdev_ftm_intg_cmdid = WMI_PDEV_FTM_INTG_CMDID, .vdev_set_keepalive_cmdid = WMI_VDEV_SET_KEEPALIVE_CMDID, .vdev_get_keepalive_cmdid = WMI_VDEV_GET_KEEPALIVE_CMDID, .force_fw_hang_cmdid = WMI_FORCE_FW_HANG_CMDID, .gpio_config_cmdid = WMI_GPIO_CONFIG_CMDID, .gpio_output_cmdid = WMI_GPIO_OUTPUT_CMDID, .pdev_get_temperature_cmdid = WMI_CMD_UNSUPPORTED, .pdev_enable_adaptive_cca_cmdid = WMI_CMD_UNSUPPORTED, .scan_update_request_cmdid = WMI_CMD_UNSUPPORTED, .vdev_standby_response_cmdid = WMI_CMD_UNSUPPORTED, .vdev_resume_response_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_add_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_evict_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_restore_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_print_all_peers_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_update_wds_entry_cmdid = WMI_CMD_UNSUPPORTED, .peer_add_proxy_sta_entry_cmdid = WMI_CMD_UNSUPPORTED, .rtt_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .oem_req_cmdid = WMI_CMD_UNSUPPORTED, .nan_cmdid = WMI_CMD_UNSUPPORTED, .vdev_ratemask_cmdid = WMI_CMD_UNSUPPORTED, .qboost_cfg_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_enable_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_set_rx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_tx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_train_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_node_config_ops_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_antenna_switch_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_ctl_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_mimogain_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_chainmsk_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_fips_cmdid = WMI_CMD_UNSUPPORTED, .tt_set_conf_cmdid = WMI_CMD_UNSUPPORTED, .fwtest_cmdid = WMI_CMD_UNSUPPORTED, .vdev_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .peer_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_cck_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_ofdm_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_reserve_ast_entry_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_nfcal_power_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ast_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_dscp_tid_map_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_filter_neighbor_rx_packets_cmdid = WMI_CMD_UNSUPPORTED, .mu_cal_start_cmdid = WMI_CMD_UNSUPPORTED, .set_cca_params_cmdid = WMI_CMD_UNSUPPORTED, .pdev_bss_chan_info_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED, .radar_found_cmdid = WMI_CMD_UNSUPPORTED, }; /* 10.X WMI cmd track */ static struct wmi_cmd_map wmi_10x_cmd_map = { .init_cmdid = WMI_10X_INIT_CMDID, .start_scan_cmdid = WMI_10X_START_SCAN_CMDID, .stop_scan_cmdid = WMI_10X_STOP_SCAN_CMDID, .scan_chan_list_cmdid = WMI_10X_SCAN_CHAN_LIST_CMDID, .scan_sch_prio_tbl_cmdid = WMI_CMD_UNSUPPORTED, .scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_regdomain_cmdid = WMI_10X_PDEV_SET_REGDOMAIN_CMDID, .pdev_set_channel_cmdid = WMI_10X_PDEV_SET_CHANNEL_CMDID, .pdev_set_param_cmdid = WMI_10X_PDEV_SET_PARAM_CMDID, .pdev_pktlog_enable_cmdid = WMI_10X_PDEV_PKTLOG_ENABLE_CMDID, .pdev_pktlog_disable_cmdid = WMI_10X_PDEV_PKTLOG_DISABLE_CMDID, .pdev_set_wmm_params_cmdid = WMI_10X_PDEV_SET_WMM_PARAMS_CMDID, .pdev_set_ht_cap_ie_cmdid = WMI_10X_PDEV_SET_HT_CAP_IE_CMDID, .pdev_set_vht_cap_ie_cmdid = WMI_10X_PDEV_SET_VHT_CAP_IE_CMDID, .pdev_set_dscp_tid_map_cmdid = WMI_10X_PDEV_SET_DSCP_TID_MAP_CMDID, .pdev_set_quiet_mode_cmdid = WMI_10X_PDEV_SET_QUIET_MODE_CMDID, .pdev_green_ap_ps_enable_cmdid = WMI_10X_PDEV_GREEN_AP_PS_ENABLE_CMDID, .pdev_get_tpc_config_cmdid = WMI_10X_PDEV_GET_TPC_CONFIG_CMDID, .pdev_set_base_macaddr_cmdid = WMI_10X_PDEV_SET_BASE_MACADDR_CMDID, .vdev_create_cmdid = WMI_10X_VDEV_CREATE_CMDID, .vdev_delete_cmdid = WMI_10X_VDEV_DELETE_CMDID, .vdev_start_request_cmdid = WMI_10X_VDEV_START_REQUEST_CMDID, .vdev_restart_request_cmdid = WMI_10X_VDEV_RESTART_REQUEST_CMDID, .vdev_up_cmdid = WMI_10X_VDEV_UP_CMDID, .vdev_stop_cmdid = WMI_10X_VDEV_STOP_CMDID, .vdev_down_cmdid = WMI_10X_VDEV_DOWN_CMDID, .vdev_set_param_cmdid = WMI_10X_VDEV_SET_PARAM_CMDID, .vdev_install_key_cmdid = WMI_10X_VDEV_INSTALL_KEY_CMDID, .peer_create_cmdid = WMI_10X_PEER_CREATE_CMDID, .peer_delete_cmdid = WMI_10X_PEER_DELETE_CMDID, .peer_flush_tids_cmdid = WMI_10X_PEER_FLUSH_TIDS_CMDID, .peer_set_param_cmdid = WMI_10X_PEER_SET_PARAM_CMDID, .peer_assoc_cmdid = WMI_10X_PEER_ASSOC_CMDID, .peer_add_wds_entry_cmdid = WMI_10X_PEER_ADD_WDS_ENTRY_CMDID, .peer_remove_wds_entry_cmdid = WMI_10X_PEER_REMOVE_WDS_ENTRY_CMDID, .peer_mcast_group_cmdid = WMI_10X_PEER_MCAST_GROUP_CMDID, .bcn_tx_cmdid = WMI_10X_BCN_TX_CMDID, .pdev_send_bcn_cmdid = WMI_10X_PDEV_SEND_BCN_CMDID, .bcn_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .bcn_filter_rx_cmdid = WMI_10X_BCN_FILTER_RX_CMDID, .prb_req_filter_rx_cmdid = WMI_10X_PRB_REQ_FILTER_RX_CMDID, .mgmt_tx_cmdid = WMI_10X_MGMT_TX_CMDID, .prb_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .addba_clear_resp_cmdid = WMI_10X_ADDBA_CLEAR_RESP_CMDID, .addba_send_cmdid = WMI_10X_ADDBA_SEND_CMDID, .addba_status_cmdid = WMI_10X_ADDBA_STATUS_CMDID, .delba_send_cmdid = WMI_10X_DELBA_SEND_CMDID, .addba_set_resp_cmdid = WMI_10X_ADDBA_SET_RESP_CMDID, .send_singleamsdu_cmdid = WMI_10X_SEND_SINGLEAMSDU_CMDID, .sta_powersave_mode_cmdid = WMI_10X_STA_POWERSAVE_MODE_CMDID, .sta_powersave_param_cmdid = WMI_10X_STA_POWERSAVE_PARAM_CMDID, .sta_mimo_ps_mode_cmdid = WMI_10X_STA_MIMO_PS_MODE_CMDID, .pdev_dfs_enable_cmdid = WMI_10X_PDEV_DFS_ENABLE_CMDID, .pdev_dfs_disable_cmdid = WMI_10X_PDEV_DFS_DISABLE_CMDID, .roam_scan_mode = WMI_10X_ROAM_SCAN_MODE, .roam_scan_rssi_threshold = WMI_10X_ROAM_SCAN_RSSI_THRESHOLD, .roam_scan_period = WMI_10X_ROAM_SCAN_PERIOD, .roam_scan_rssi_change_threshold = WMI_10X_ROAM_SCAN_RSSI_CHANGE_THRESHOLD, .roam_ap_profile = WMI_10X_ROAM_AP_PROFILE, .ofl_scan_add_ap_profile = WMI_10X_OFL_SCAN_ADD_AP_PROFILE, .ofl_scan_remove_ap_profile = WMI_10X_OFL_SCAN_REMOVE_AP_PROFILE, .ofl_scan_period = WMI_10X_OFL_SCAN_PERIOD, .p2p_dev_set_device_info = WMI_10X_P2P_DEV_SET_DEVICE_INFO, .p2p_dev_set_discoverability = WMI_10X_P2P_DEV_SET_DISCOVERABILITY, .p2p_go_set_beacon_ie = WMI_10X_P2P_GO_SET_BEACON_IE, .p2p_go_set_probe_resp_ie = WMI_10X_P2P_GO_SET_PROBE_RESP_IE, .p2p_set_vendor_ie_data_cmdid = WMI_CMD_UNSUPPORTED, .ap_ps_peer_param_cmdid = WMI_10X_AP_PS_PEER_PARAM_CMDID, .ap_ps_peer_uapsd_coex_cmdid = WMI_CMD_UNSUPPORTED, .peer_rate_retry_sched_cmdid = WMI_10X_PEER_RATE_RETRY_SCHED_CMDID, .wlan_profile_trigger_cmdid = WMI_10X_WLAN_PROFILE_TRIGGER_CMDID, .wlan_profile_set_hist_intvl_cmdid = WMI_10X_WLAN_PROFILE_SET_HIST_INTVL_CMDID, .wlan_profile_get_profile_data_cmdid = WMI_10X_WLAN_PROFILE_GET_PROFILE_DATA_CMDID, .wlan_profile_enable_profile_id_cmdid = WMI_10X_WLAN_PROFILE_ENABLE_PROFILE_ID_CMDID, .wlan_profile_list_profile_id_cmdid = WMI_10X_WLAN_PROFILE_LIST_PROFILE_ID_CMDID, .pdev_suspend_cmdid = WMI_10X_PDEV_SUSPEND_CMDID, .pdev_resume_cmdid = WMI_10X_PDEV_RESUME_CMDID, .add_bcn_filter_cmdid = WMI_10X_ADD_BCN_FILTER_CMDID, .rmv_bcn_filter_cmdid = WMI_10X_RMV_BCN_FILTER_CMDID, .wow_add_wake_pattern_cmdid = WMI_10X_WOW_ADD_WAKE_PATTERN_CMDID, .wow_del_wake_pattern_cmdid = WMI_10X_WOW_DEL_WAKE_PATTERN_CMDID, .wow_enable_disable_wake_event_cmdid = WMI_10X_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID, .wow_enable_cmdid = WMI_10X_WOW_ENABLE_CMDID, .wow_hostwakeup_from_sleep_cmdid = WMI_10X_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID, .rtt_measreq_cmdid = WMI_10X_RTT_MEASREQ_CMDID, .rtt_tsf_cmdid = WMI_10X_RTT_TSF_CMDID, .vdev_spectral_scan_configure_cmdid = WMI_10X_VDEV_SPECTRAL_SCAN_CONFIGURE_CMDID, .vdev_spectral_scan_enable_cmdid = WMI_10X_VDEV_SPECTRAL_SCAN_ENABLE_CMDID, .request_stats_cmdid = WMI_10X_REQUEST_STATS_CMDID, .set_arp_ns_offload_cmdid = WMI_CMD_UNSUPPORTED, .network_list_offload_config_cmdid = WMI_CMD_UNSUPPORTED, .gtk_offload_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_enable_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_chanswitch_cmdid = WMI_CMD_UNSUPPORTED, .chatter_set_mode_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_addba_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_delba_cmdid = WMI_CMD_UNSUPPORTED, .sta_dtim_ps_method_cmdid = WMI_CMD_UNSUPPORTED, .sta_uapsd_auto_trig_cmdid = WMI_CMD_UNSUPPORTED, .sta_keepalive_cmd = WMI_CMD_UNSUPPORTED, .echo_cmdid = WMI_10X_ECHO_CMDID, .pdev_utf_cmdid = WMI_10X_PDEV_UTF_CMDID, .dbglog_cfg_cmdid = WMI_10X_DBGLOG_CFG_CMDID, .pdev_qvit_cmdid = WMI_10X_PDEV_QVIT_CMDID, .pdev_ftm_intg_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .force_fw_hang_cmdid = WMI_CMD_UNSUPPORTED, .gpio_config_cmdid = WMI_10X_GPIO_CONFIG_CMDID, .gpio_output_cmdid = WMI_10X_GPIO_OUTPUT_CMDID, .pdev_get_temperature_cmdid = WMI_CMD_UNSUPPORTED, .pdev_enable_adaptive_cca_cmdid = WMI_CMD_UNSUPPORTED, .scan_update_request_cmdid = WMI_CMD_UNSUPPORTED, .vdev_standby_response_cmdid = WMI_CMD_UNSUPPORTED, .vdev_resume_response_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_add_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_evict_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_restore_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_print_all_peers_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_update_wds_entry_cmdid = WMI_CMD_UNSUPPORTED, .peer_add_proxy_sta_entry_cmdid = WMI_CMD_UNSUPPORTED, .rtt_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .oem_req_cmdid = WMI_CMD_UNSUPPORTED, .nan_cmdid = WMI_CMD_UNSUPPORTED, .vdev_ratemask_cmdid = WMI_CMD_UNSUPPORTED, .qboost_cfg_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_enable_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_set_rx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_tx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_train_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_node_config_ops_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_antenna_switch_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_ctl_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_mimogain_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_chainmsk_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_fips_cmdid = WMI_CMD_UNSUPPORTED, .tt_set_conf_cmdid = WMI_CMD_UNSUPPORTED, .fwtest_cmdid = WMI_CMD_UNSUPPORTED, .vdev_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .peer_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_cck_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_ofdm_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_reserve_ast_entry_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_nfcal_power_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ast_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_dscp_tid_map_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_filter_neighbor_rx_packets_cmdid = WMI_CMD_UNSUPPORTED, .mu_cal_start_cmdid = WMI_CMD_UNSUPPORTED, .set_cca_params_cmdid = WMI_CMD_UNSUPPORTED, .pdev_bss_chan_info_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED, .radar_found_cmdid = WMI_CMD_UNSUPPORTED, }; /* 10.2.4 WMI cmd track */ static struct wmi_cmd_map wmi_10_2_4_cmd_map = { .init_cmdid = WMI_10_2_INIT_CMDID, .start_scan_cmdid = WMI_10_2_START_SCAN_CMDID, .stop_scan_cmdid = WMI_10_2_STOP_SCAN_CMDID, .scan_chan_list_cmdid = WMI_10_2_SCAN_CHAN_LIST_CMDID, .scan_sch_prio_tbl_cmdid = WMI_CMD_UNSUPPORTED, .scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_regdomain_cmdid = WMI_10_2_PDEV_SET_REGDOMAIN_CMDID, .pdev_set_channel_cmdid = WMI_10_2_PDEV_SET_CHANNEL_CMDID, .pdev_set_param_cmdid = WMI_10_2_PDEV_SET_PARAM_CMDID, .pdev_pktlog_enable_cmdid = WMI_10_2_PDEV_PKTLOG_ENABLE_CMDID, .pdev_pktlog_disable_cmdid = WMI_10_2_PDEV_PKTLOG_DISABLE_CMDID, .pdev_set_wmm_params_cmdid = WMI_10_2_PDEV_SET_WMM_PARAMS_CMDID, .pdev_set_ht_cap_ie_cmdid = WMI_10_2_PDEV_SET_HT_CAP_IE_CMDID, .pdev_set_vht_cap_ie_cmdid = WMI_10_2_PDEV_SET_VHT_CAP_IE_CMDID, .pdev_set_quiet_mode_cmdid = WMI_10_2_PDEV_SET_QUIET_MODE_CMDID, .pdev_green_ap_ps_enable_cmdid = WMI_10_2_PDEV_GREEN_AP_PS_ENABLE_CMDID, .pdev_get_tpc_config_cmdid = WMI_10_2_PDEV_GET_TPC_CONFIG_CMDID, .pdev_set_base_macaddr_cmdid = WMI_10_2_PDEV_SET_BASE_MACADDR_CMDID, .vdev_create_cmdid = WMI_10_2_VDEV_CREATE_CMDID, .vdev_delete_cmdid = WMI_10_2_VDEV_DELETE_CMDID, .vdev_start_request_cmdid = WMI_10_2_VDEV_START_REQUEST_CMDID, .vdev_restart_request_cmdid = WMI_10_2_VDEV_RESTART_REQUEST_CMDID, .vdev_up_cmdid = WMI_10_2_VDEV_UP_CMDID, .vdev_stop_cmdid = WMI_10_2_VDEV_STOP_CMDID, .vdev_down_cmdid = WMI_10_2_VDEV_DOWN_CMDID, .vdev_set_param_cmdid = WMI_10_2_VDEV_SET_PARAM_CMDID, .vdev_install_key_cmdid = WMI_10_2_VDEV_INSTALL_KEY_CMDID, .peer_create_cmdid = WMI_10_2_PEER_CREATE_CMDID, .peer_delete_cmdid = WMI_10_2_PEER_DELETE_CMDID, .peer_flush_tids_cmdid = WMI_10_2_PEER_FLUSH_TIDS_CMDID, .peer_set_param_cmdid = WMI_10_2_PEER_SET_PARAM_CMDID, .peer_assoc_cmdid = WMI_10_2_PEER_ASSOC_CMDID, .peer_add_wds_entry_cmdid = WMI_10_2_PEER_ADD_WDS_ENTRY_CMDID, .peer_remove_wds_entry_cmdid = WMI_10_2_PEER_REMOVE_WDS_ENTRY_CMDID, .peer_mcast_group_cmdid = WMI_10_2_PEER_MCAST_GROUP_CMDID, .bcn_tx_cmdid = WMI_10_2_BCN_TX_CMDID, .pdev_send_bcn_cmdid = WMI_10_2_PDEV_SEND_BCN_CMDID, .bcn_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .bcn_filter_rx_cmdid = WMI_10_2_BCN_FILTER_RX_CMDID, .prb_req_filter_rx_cmdid = WMI_10_2_PRB_REQ_FILTER_RX_CMDID, .mgmt_tx_cmdid = WMI_10_2_MGMT_TX_CMDID, .prb_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .addba_clear_resp_cmdid = WMI_10_2_ADDBA_CLEAR_RESP_CMDID, .addba_send_cmdid = WMI_10_2_ADDBA_SEND_CMDID, .addba_status_cmdid = WMI_10_2_ADDBA_STATUS_CMDID, .delba_send_cmdid = WMI_10_2_DELBA_SEND_CMDID, .addba_set_resp_cmdid = WMI_10_2_ADDBA_SET_RESP_CMDID, .send_singleamsdu_cmdid = WMI_10_2_SEND_SINGLEAMSDU_CMDID, .sta_powersave_mode_cmdid = WMI_10_2_STA_POWERSAVE_MODE_CMDID, .sta_powersave_param_cmdid = WMI_10_2_STA_POWERSAVE_PARAM_CMDID, .sta_mimo_ps_mode_cmdid = WMI_10_2_STA_MIMO_PS_MODE_CMDID, .pdev_dfs_enable_cmdid = WMI_10_2_PDEV_DFS_ENABLE_CMDID, .pdev_dfs_disable_cmdid = WMI_10_2_PDEV_DFS_DISABLE_CMDID, .roam_scan_mode = WMI_10_2_ROAM_SCAN_MODE, .roam_scan_rssi_threshold = WMI_10_2_ROAM_SCAN_RSSI_THRESHOLD, .roam_scan_period = WMI_10_2_ROAM_SCAN_PERIOD, .roam_scan_rssi_change_threshold = WMI_10_2_ROAM_SCAN_RSSI_CHANGE_THRESHOLD, .roam_ap_profile = WMI_10_2_ROAM_AP_PROFILE, .ofl_scan_add_ap_profile = WMI_10_2_OFL_SCAN_ADD_AP_PROFILE, .ofl_scan_remove_ap_profile = WMI_10_2_OFL_SCAN_REMOVE_AP_PROFILE, .ofl_scan_period = WMI_10_2_OFL_SCAN_PERIOD, .p2p_dev_set_device_info = WMI_10_2_P2P_DEV_SET_DEVICE_INFO, .p2p_dev_set_discoverability = WMI_10_2_P2P_DEV_SET_DISCOVERABILITY, .p2p_go_set_beacon_ie = WMI_10_2_P2P_GO_SET_BEACON_IE, .p2p_go_set_probe_resp_ie = WMI_10_2_P2P_GO_SET_PROBE_RESP_IE, .p2p_set_vendor_ie_data_cmdid = WMI_CMD_UNSUPPORTED, .ap_ps_peer_param_cmdid = WMI_10_2_AP_PS_PEER_PARAM_CMDID, .ap_ps_peer_uapsd_coex_cmdid = WMI_CMD_UNSUPPORTED, .peer_rate_retry_sched_cmdid = WMI_10_2_PEER_RATE_RETRY_SCHED_CMDID, .wlan_profile_trigger_cmdid = WMI_10_2_WLAN_PROFILE_TRIGGER_CMDID, .wlan_profile_set_hist_intvl_cmdid = WMI_10_2_WLAN_PROFILE_SET_HIST_INTVL_CMDID, .wlan_profile_get_profile_data_cmdid = WMI_10_2_WLAN_PROFILE_GET_PROFILE_DATA_CMDID, .wlan_profile_enable_profile_id_cmdid = WMI_10_2_WLAN_PROFILE_ENABLE_PROFILE_ID_CMDID, .wlan_profile_list_profile_id_cmdid = WMI_10_2_WLAN_PROFILE_LIST_PROFILE_ID_CMDID, .pdev_suspend_cmdid = WMI_10_2_PDEV_SUSPEND_CMDID, .pdev_resume_cmdid = WMI_10_2_PDEV_RESUME_CMDID, .add_bcn_filter_cmdid = WMI_10_2_ADD_BCN_FILTER_CMDID, .rmv_bcn_filter_cmdid = WMI_10_2_RMV_BCN_FILTER_CMDID, .wow_add_wake_pattern_cmdid = WMI_10_2_WOW_ADD_WAKE_PATTERN_CMDID, .wow_del_wake_pattern_cmdid = WMI_10_2_WOW_DEL_WAKE_PATTERN_CMDID, .wow_enable_disable_wake_event_cmdid = WMI_10_2_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID, .wow_enable_cmdid = WMI_10_2_WOW_ENABLE_CMDID, .wow_hostwakeup_from_sleep_cmdid = WMI_10_2_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID, .rtt_measreq_cmdid = WMI_10_2_RTT_MEASREQ_CMDID, .rtt_tsf_cmdid = WMI_10_2_RTT_TSF_CMDID, .vdev_spectral_scan_configure_cmdid = WMI_10_2_VDEV_SPECTRAL_SCAN_CONFIGURE_CMDID, .vdev_spectral_scan_enable_cmdid = WMI_10_2_VDEV_SPECTRAL_SCAN_ENABLE_CMDID, .request_stats_cmdid = WMI_10_2_REQUEST_STATS_CMDID, .set_arp_ns_offload_cmdid = WMI_CMD_UNSUPPORTED, .network_list_offload_config_cmdid = WMI_CMD_UNSUPPORTED, .gtk_offload_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_enable_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_chanswitch_cmdid = WMI_CMD_UNSUPPORTED, .chatter_set_mode_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_addba_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_delba_cmdid = WMI_CMD_UNSUPPORTED, .sta_dtim_ps_method_cmdid = WMI_CMD_UNSUPPORTED, .sta_uapsd_auto_trig_cmdid = WMI_CMD_UNSUPPORTED, .sta_keepalive_cmd = WMI_CMD_UNSUPPORTED, .echo_cmdid = WMI_10_2_ECHO_CMDID, .pdev_utf_cmdid = WMI_10_2_PDEV_UTF_CMDID, .dbglog_cfg_cmdid = WMI_10_2_DBGLOG_CFG_CMDID, .pdev_qvit_cmdid = WMI_10_2_PDEV_QVIT_CMDID, .pdev_ftm_intg_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .force_fw_hang_cmdid = WMI_CMD_UNSUPPORTED, .gpio_config_cmdid = WMI_10_2_GPIO_CONFIG_CMDID, .gpio_output_cmdid = WMI_10_2_GPIO_OUTPUT_CMDID, .pdev_get_temperature_cmdid = WMI_10_2_PDEV_GET_TEMPERATURE_CMDID, .pdev_enable_adaptive_cca_cmdid = WMI_10_2_SET_CCA_PARAMS, .scan_update_request_cmdid = WMI_CMD_UNSUPPORTED, .vdev_standby_response_cmdid = WMI_CMD_UNSUPPORTED, .vdev_resume_response_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_add_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_evict_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_restore_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_print_all_peers_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_update_wds_entry_cmdid = WMI_CMD_UNSUPPORTED, .peer_add_proxy_sta_entry_cmdid = WMI_CMD_UNSUPPORTED, .rtt_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .oem_req_cmdid = WMI_CMD_UNSUPPORTED, .nan_cmdid = WMI_CMD_UNSUPPORTED, .vdev_ratemask_cmdid = WMI_CMD_UNSUPPORTED, .qboost_cfg_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_enable_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_set_rx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_tx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_train_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_node_config_ops_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_antenna_switch_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_ctl_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_mimogain_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_chainmsk_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_fips_cmdid = WMI_CMD_UNSUPPORTED, .tt_set_conf_cmdid = WMI_CMD_UNSUPPORTED, .fwtest_cmdid = WMI_CMD_UNSUPPORTED, .vdev_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .peer_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_cck_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_ofdm_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_reserve_ast_entry_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_nfcal_power_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ast_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_dscp_tid_map_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_filter_neighbor_rx_packets_cmdid = WMI_CMD_UNSUPPORTED, .mu_cal_start_cmdid = WMI_CMD_UNSUPPORTED, .set_cca_params_cmdid = WMI_CMD_UNSUPPORTED, .pdev_bss_chan_info_request_cmdid = WMI_10_2_PDEV_BSS_CHAN_INFO_REQUEST_CMDID, .pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED, .radar_found_cmdid = WMI_CMD_UNSUPPORTED, .set_bb_timing_cmdid = WMI_10_2_PDEV_SET_BB_TIMING_CONFIG_CMDID, }; /* 10.4 WMI cmd track */ static struct wmi_cmd_map wmi_10_4_cmd_map = { .init_cmdid = WMI_10_4_INIT_CMDID, .start_scan_cmdid = WMI_10_4_START_SCAN_CMDID, .stop_scan_cmdid = WMI_10_4_STOP_SCAN_CMDID, .scan_chan_list_cmdid = WMI_10_4_SCAN_CHAN_LIST_CMDID, .scan_sch_prio_tbl_cmdid = WMI_10_4_SCAN_SCH_PRIO_TBL_CMDID, .scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_regdomain_cmdid = WMI_10_4_PDEV_SET_REGDOMAIN_CMDID, .pdev_set_channel_cmdid = WMI_10_4_PDEV_SET_CHANNEL_CMDID, .pdev_set_param_cmdid = WMI_10_4_PDEV_SET_PARAM_CMDID, .pdev_pktlog_enable_cmdid = WMI_10_4_PDEV_PKTLOG_ENABLE_CMDID, .pdev_pktlog_disable_cmdid = WMI_10_4_PDEV_PKTLOG_DISABLE_CMDID, .pdev_set_wmm_params_cmdid = WMI_10_4_PDEV_SET_WMM_PARAMS_CMDID, .pdev_set_ht_cap_ie_cmdid = WMI_10_4_PDEV_SET_HT_CAP_IE_CMDID, .pdev_set_vht_cap_ie_cmdid = WMI_10_4_PDEV_SET_VHT_CAP_IE_CMDID, .pdev_set_dscp_tid_map_cmdid = WMI_10_4_PDEV_SET_DSCP_TID_MAP_CMDID, .pdev_set_quiet_mode_cmdid = WMI_10_4_PDEV_SET_QUIET_MODE_CMDID, .pdev_green_ap_ps_enable_cmdid = WMI_10_4_PDEV_GREEN_AP_PS_ENABLE_CMDID, .pdev_get_tpc_config_cmdid = WMI_10_4_PDEV_GET_TPC_CONFIG_CMDID, .pdev_set_base_macaddr_cmdid = WMI_10_4_PDEV_SET_BASE_MACADDR_CMDID, .vdev_create_cmdid = WMI_10_4_VDEV_CREATE_CMDID, .vdev_delete_cmdid = WMI_10_4_VDEV_DELETE_CMDID, .vdev_start_request_cmdid = WMI_10_4_VDEV_START_REQUEST_CMDID, .vdev_restart_request_cmdid = WMI_10_4_VDEV_RESTART_REQUEST_CMDID, .vdev_up_cmdid = WMI_10_4_VDEV_UP_CMDID, .vdev_stop_cmdid = WMI_10_4_VDEV_STOP_CMDID, .vdev_down_cmdid = WMI_10_4_VDEV_DOWN_CMDID, .vdev_set_param_cmdid = WMI_10_4_VDEV_SET_PARAM_CMDID, .vdev_install_key_cmdid = WMI_10_4_VDEV_INSTALL_KEY_CMDID, .peer_create_cmdid = WMI_10_4_PEER_CREATE_CMDID, .peer_delete_cmdid = WMI_10_4_PEER_DELETE_CMDID, .peer_flush_tids_cmdid = WMI_10_4_PEER_FLUSH_TIDS_CMDID, .peer_set_param_cmdid = WMI_10_4_PEER_SET_PARAM_CMDID, .peer_assoc_cmdid = WMI_10_4_PEER_ASSOC_CMDID, .peer_add_wds_entry_cmdid = WMI_10_4_PEER_ADD_WDS_ENTRY_CMDID, .peer_remove_wds_entry_cmdid = WMI_10_4_PEER_REMOVE_WDS_ENTRY_CMDID, .peer_mcast_group_cmdid = WMI_10_4_PEER_MCAST_GROUP_CMDID, .bcn_tx_cmdid = WMI_10_4_BCN_TX_CMDID, .pdev_send_bcn_cmdid = WMI_10_4_PDEV_SEND_BCN_CMDID, .bcn_tmpl_cmdid = WMI_10_4_BCN_PRB_TMPL_CMDID, .bcn_filter_rx_cmdid = WMI_10_4_BCN_FILTER_RX_CMDID, .prb_req_filter_rx_cmdid = WMI_10_4_PRB_REQ_FILTER_RX_CMDID, .mgmt_tx_cmdid = WMI_10_4_MGMT_TX_CMDID, .prb_tmpl_cmdid = WMI_10_4_PRB_TMPL_CMDID, .addba_clear_resp_cmdid = WMI_10_4_ADDBA_CLEAR_RESP_CMDID, .addba_send_cmdid = WMI_10_4_ADDBA_SEND_CMDID, .addba_status_cmdid = WMI_10_4_ADDBA_STATUS_CMDID, .delba_send_cmdid = WMI_10_4_DELBA_SEND_CMDID, .addba_set_resp_cmdid = WMI_10_4_ADDBA_SET_RESP_CMDID, .send_singleamsdu_cmdid = WMI_10_4_SEND_SINGLEAMSDU_CMDID, .sta_powersave_mode_cmdid = WMI_10_4_STA_POWERSAVE_MODE_CMDID, .sta_powersave_param_cmdid = WMI_10_4_STA_POWERSAVE_PARAM_CMDID, .sta_mimo_ps_mode_cmdid = WMI_10_4_STA_MIMO_PS_MODE_CMDID, .pdev_dfs_enable_cmdid = WMI_10_4_PDEV_DFS_ENABLE_CMDID, .pdev_dfs_disable_cmdid = WMI_10_4_PDEV_DFS_DISABLE_CMDID, .roam_scan_mode = WMI_10_4_ROAM_SCAN_MODE, .roam_scan_rssi_threshold = WMI_10_4_ROAM_SCAN_RSSI_THRESHOLD, .roam_scan_period = WMI_10_4_ROAM_SCAN_PERIOD, .roam_scan_rssi_change_threshold = WMI_10_4_ROAM_SCAN_RSSI_CHANGE_THRESHOLD, .roam_ap_profile = WMI_10_4_ROAM_AP_PROFILE, .ofl_scan_add_ap_profile = WMI_10_4_OFL_SCAN_ADD_AP_PROFILE, .ofl_scan_remove_ap_profile = WMI_10_4_OFL_SCAN_REMOVE_AP_PROFILE, .ofl_scan_period = WMI_10_4_OFL_SCAN_PERIOD, .p2p_dev_set_device_info = WMI_10_4_P2P_DEV_SET_DEVICE_INFO, .p2p_dev_set_discoverability = WMI_10_4_P2P_DEV_SET_DISCOVERABILITY, .p2p_go_set_beacon_ie = WMI_10_4_P2P_GO_SET_BEACON_IE, .p2p_go_set_probe_resp_ie = WMI_10_4_P2P_GO_SET_PROBE_RESP_IE, .p2p_set_vendor_ie_data_cmdid = WMI_10_4_P2P_SET_VENDOR_IE_DATA_CMDID, .ap_ps_peer_param_cmdid = WMI_10_4_AP_PS_PEER_PARAM_CMDID, .ap_ps_peer_uapsd_coex_cmdid = WMI_10_4_AP_PS_PEER_UAPSD_COEX_CMDID, .peer_rate_retry_sched_cmdid = WMI_10_4_PEER_RATE_RETRY_SCHED_CMDID, .wlan_profile_trigger_cmdid = WMI_10_4_WLAN_PROFILE_TRIGGER_CMDID, .wlan_profile_set_hist_intvl_cmdid = WMI_10_4_WLAN_PROFILE_SET_HIST_INTVL_CMDID, .wlan_profile_get_profile_data_cmdid = WMI_10_4_WLAN_PROFILE_GET_PROFILE_DATA_CMDID, .wlan_profile_enable_profile_id_cmdid = WMI_10_4_WLAN_PROFILE_ENABLE_PROFILE_ID_CMDID, .wlan_profile_list_profile_id_cmdid = WMI_10_4_WLAN_PROFILE_LIST_PROFILE_ID_CMDID, .pdev_suspend_cmdid = WMI_10_4_PDEV_SUSPEND_CMDID, .pdev_resume_cmdid = WMI_10_4_PDEV_RESUME_CMDID, .add_bcn_filter_cmdid = WMI_10_4_ADD_BCN_FILTER_CMDID, .rmv_bcn_filter_cmdid = WMI_10_4_RMV_BCN_FILTER_CMDID, .wow_add_wake_pattern_cmdid = WMI_10_4_WOW_ADD_WAKE_PATTERN_CMDID, .wow_del_wake_pattern_cmdid = WMI_10_4_WOW_DEL_WAKE_PATTERN_CMDID, .wow_enable_disable_wake_event_cmdid = WMI_10_4_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID, .wow_enable_cmdid = WMI_10_4_WOW_ENABLE_CMDID, .wow_hostwakeup_from_sleep_cmdid = WMI_10_4_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID, .rtt_measreq_cmdid = WMI_10_4_RTT_MEASREQ_CMDID, .rtt_tsf_cmdid = WMI_10_4_RTT_TSF_CMDID, .vdev_spectral_scan_configure_cmdid = WMI_10_4_VDEV_SPECTRAL_SCAN_CONFIGURE_CMDID, .vdev_spectral_scan_enable_cmdid = WMI_10_4_VDEV_SPECTRAL_SCAN_ENABLE_CMDID, .request_stats_cmdid = WMI_10_4_REQUEST_STATS_CMDID, .set_arp_ns_offload_cmdid = WMI_CMD_UNSUPPORTED, .network_list_offload_config_cmdid = WMI_CMD_UNSUPPORTED, .gtk_offload_cmdid = WMI_10_4_GTK_OFFLOAD_CMDID, .csa_offload_enable_cmdid = WMI_10_4_CSA_OFFLOAD_ENABLE_CMDID, .csa_offload_chanswitch_cmdid = WMI_10_4_CSA_OFFLOAD_CHANSWITCH_CMDID, .chatter_set_mode_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_addba_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_delba_cmdid = WMI_CMD_UNSUPPORTED, .sta_dtim_ps_method_cmdid = WMI_CMD_UNSUPPORTED, .sta_uapsd_auto_trig_cmdid = WMI_CMD_UNSUPPORTED, .sta_keepalive_cmd = WMI_CMD_UNSUPPORTED, .echo_cmdid = WMI_10_4_ECHO_CMDID, .pdev_utf_cmdid = WMI_10_4_PDEV_UTF_CMDID, .dbglog_cfg_cmdid = WMI_10_4_DBGLOG_CFG_CMDID, .pdev_qvit_cmdid = WMI_10_4_PDEV_QVIT_CMDID, .pdev_ftm_intg_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_keepalive_cmdid = WMI_10_4_VDEV_SET_KEEPALIVE_CMDID, .vdev_get_keepalive_cmdid = WMI_10_4_VDEV_GET_KEEPALIVE_CMDID, .force_fw_hang_cmdid = WMI_10_4_FORCE_FW_HANG_CMDID, .gpio_config_cmdid = WMI_10_4_GPIO_CONFIG_CMDID, .gpio_output_cmdid = WMI_10_4_GPIO_OUTPUT_CMDID, .pdev_get_temperature_cmdid = WMI_10_4_PDEV_GET_TEMPERATURE_CMDID, .vdev_set_wmm_params_cmdid = WMI_CMD_UNSUPPORTED, .adaptive_qcs_cmdid = WMI_CMD_UNSUPPORTED, .scan_update_request_cmdid = WMI_10_4_SCAN_UPDATE_REQUEST_CMDID, .vdev_standby_response_cmdid = WMI_10_4_VDEV_STANDBY_RESPONSE_CMDID, .vdev_resume_response_cmdid = WMI_10_4_VDEV_RESUME_RESPONSE_CMDID, .wlan_peer_caching_add_peer_cmdid = WMI_10_4_WLAN_PEER_CACHING_ADD_PEER_CMDID, .wlan_peer_caching_evict_peer_cmdid = WMI_10_4_WLAN_PEER_CACHING_EVICT_PEER_CMDID, .wlan_peer_caching_restore_peer_cmdid = WMI_10_4_WLAN_PEER_CACHING_RESTORE_PEER_CMDID, .wlan_peer_caching_print_all_peers_info_cmdid = WMI_10_4_WLAN_PEER_CACHING_PRINT_ALL_PEERS_INFO_CMDID, .peer_update_wds_entry_cmdid = WMI_10_4_PEER_UPDATE_WDS_ENTRY_CMDID, .peer_add_proxy_sta_entry_cmdid = WMI_10_4_PEER_ADD_PROXY_STA_ENTRY_CMDID, .rtt_keepalive_cmdid = WMI_10_4_RTT_KEEPALIVE_CMDID, .oem_req_cmdid = WMI_10_4_OEM_REQ_CMDID, .nan_cmdid = WMI_10_4_NAN_CMDID, .vdev_ratemask_cmdid = WMI_10_4_VDEV_RATEMASK_CMDID, .qboost_cfg_cmdid = WMI_10_4_QBOOST_CFG_CMDID, .pdev_smart_ant_enable_cmdid = WMI_10_4_PDEV_SMART_ANT_ENABLE_CMDID, .pdev_smart_ant_set_rx_antenna_cmdid = WMI_10_4_PDEV_SMART_ANT_SET_RX_ANTENNA_CMDID, .peer_smart_ant_set_tx_antenna_cmdid = WMI_10_4_PEER_SMART_ANT_SET_TX_ANTENNA_CMDID, .peer_smart_ant_set_train_info_cmdid = WMI_10_4_PEER_SMART_ANT_SET_TRAIN_INFO_CMDID, .peer_smart_ant_set_node_config_ops_cmdid = WMI_10_4_PEER_SMART_ANT_SET_NODE_CONFIG_OPS_CMDID, .pdev_set_antenna_switch_table_cmdid = WMI_10_4_PDEV_SET_ANTENNA_SWITCH_TABLE_CMDID, .pdev_set_ctl_table_cmdid = WMI_10_4_PDEV_SET_CTL_TABLE_CMDID, .pdev_set_mimogain_table_cmdid = WMI_10_4_PDEV_SET_MIMOGAIN_TABLE_CMDID, .pdev_ratepwr_table_cmdid = WMI_10_4_PDEV_RATEPWR_TABLE_CMDID, .pdev_ratepwr_chainmsk_table_cmdid = WMI_10_4_PDEV_RATEPWR_CHAINMSK_TABLE_CMDID, .pdev_fips_cmdid = WMI_10_4_PDEV_FIPS_CMDID, .tt_set_conf_cmdid = WMI_10_4_TT_SET_CONF_CMDID, .fwtest_cmdid = WMI_10_4_FWTEST_CMDID, .vdev_atf_request_cmdid = WMI_10_4_VDEV_ATF_REQUEST_CMDID, .peer_atf_request_cmdid = WMI_10_4_PEER_ATF_REQUEST_CMDID, .pdev_get_ani_cck_config_cmdid = WMI_10_4_PDEV_GET_ANI_CCK_CONFIG_CMDID, .pdev_get_ani_ofdm_config_cmdid = WMI_10_4_PDEV_GET_ANI_OFDM_CONFIG_CMDID, .pdev_reserve_ast_entry_cmdid = WMI_10_4_PDEV_RESERVE_AST_ENTRY_CMDID, .pdev_get_nfcal_power_cmdid = WMI_10_4_PDEV_GET_NFCAL_POWER_CMDID, .pdev_get_tpc_cmdid = WMI_10_4_PDEV_GET_TPC_CMDID, .pdev_get_ast_info_cmdid = WMI_10_4_PDEV_GET_AST_INFO_CMDID, .vdev_set_dscp_tid_map_cmdid = WMI_10_4_VDEV_SET_DSCP_TID_MAP_CMDID, .pdev_get_info_cmdid = WMI_10_4_PDEV_GET_INFO_CMDID, .vdev_get_info_cmdid = WMI_10_4_VDEV_GET_INFO_CMDID, .vdev_filter_neighbor_rx_packets_cmdid = WMI_10_4_VDEV_FILTER_NEIGHBOR_RX_PACKETS_CMDID, .mu_cal_start_cmdid = WMI_10_4_MU_CAL_START_CMDID, .set_cca_params_cmdid = WMI_10_4_SET_CCA_PARAMS_CMDID, .pdev_bss_chan_info_request_cmdid = WMI_10_4_PDEV_BSS_CHAN_INFO_REQUEST_CMDID, .ext_resource_cfg_cmdid = WMI_10_4_EXT_RESOURCE_CFG_CMDID, .vdev_set_ie_cmdid = WMI_10_4_VDEV_SET_IE_CMDID, .set_lteu_config_cmdid = WMI_10_4_SET_LTEU_CONFIG_CMDID, .atf_ssid_grouping_request_cmdid = WMI_10_4_ATF_SSID_GROUPING_REQUEST_CMDID, .peer_atf_ext_request_cmdid = WMI_10_4_PEER_ATF_EXT_REQUEST_CMDID, .set_periodic_channel_stats_cfg_cmdid = WMI_10_4_SET_PERIODIC_CHANNEL_STATS_CONFIG, .peer_bwf_request_cmdid = WMI_10_4_PEER_BWF_REQUEST_CMDID, .btcoex_cfg_cmdid = WMI_10_4_BTCOEX_CFG_CMDID, .peer_tx_mu_txmit_count_cmdid = WMI_10_4_PEER_TX_MU_TXMIT_COUNT_CMDID, .peer_tx_mu_txmit_rstcnt_cmdid = WMI_10_4_PEER_TX_MU_TXMIT_RSTCNT_CMDID, .peer_gid_userpos_list_cmdid = WMI_10_4_PEER_GID_USERPOS_LIST_CMDID, .pdev_check_cal_version_cmdid = WMI_10_4_PDEV_CHECK_CAL_VERSION_CMDID, .coex_version_cfg_cmid = WMI_10_4_COEX_VERSION_CFG_CMID, .pdev_get_rx_filter_cmdid = WMI_10_4_PDEV_GET_RX_FILTER_CMDID, .pdev_extended_nss_cfg_cmdid = WMI_10_4_PDEV_EXTENDED_NSS_CFG_CMDID, .vdev_set_scan_nac_rssi_cmdid = WMI_10_4_VDEV_SET_SCAN_NAC_RSSI_CMDID, .prog_gpio_band_select_cmdid = WMI_10_4_PROG_GPIO_BAND_SELECT_CMDID, .config_smart_logging_cmdid = WMI_10_4_CONFIG_SMART_LOGGING_CMDID, .debug_fatal_condition_cmdid = WMI_10_4_DEBUG_FATAL_CONDITION_CMDID, .get_tsf_timer_cmdid = WMI_10_4_GET_TSF_TIMER_CMDID, .pdev_get_tpc_table_cmdid = WMI_10_4_PDEV_GET_TPC_TABLE_CMDID, .vdev_sifs_trigger_time_cmdid = WMI_10_4_VDEV_SIFS_TRIGGER_TIME_CMDID, .pdev_wds_entry_list_cmdid = WMI_10_4_PDEV_WDS_ENTRY_LIST_CMDID, .tdls_set_state_cmdid = WMI_10_4_TDLS_SET_STATE_CMDID, .tdls_peer_update_cmdid = WMI_10_4_TDLS_PEER_UPDATE_CMDID, .tdls_set_offchan_mode_cmdid = WMI_10_4_TDLS_SET_OFFCHAN_MODE_CMDID, .radar_found_cmdid = WMI_10_4_RADAR_FOUND_CMDID, .per_peer_per_tid_config_cmdid = WMI_10_4_PER_PEER_PER_TID_CONFIG_CMDID, }; static struct wmi_peer_param_map wmi_peer_param_map = { .smps_state = WMI_PEER_SMPS_STATE, .ampdu = WMI_PEER_AMPDU, .authorize = WMI_PEER_AUTHORIZE, .chan_width = WMI_PEER_CHAN_WIDTH, .nss = WMI_PEER_NSS, .use_4addr = WMI_PEER_USE_4ADDR, .use_fixed_power = WMI_PEER_USE_FIXED_PWR, .debug = WMI_PEER_DEBUG, .phymode = WMI_PEER_PHYMODE, .dummy_var = WMI_PEER_DUMMY_VAR, }; /* MAIN WMI VDEV param map */ static struct wmi_vdev_param_map wmi_vdev_param_map = { .rts_threshold = WMI_VDEV_PARAM_RTS_THRESHOLD, .fragmentation_threshold = WMI_VDEV_PARAM_FRAGMENTATION_THRESHOLD, .beacon_interval = WMI_VDEV_PARAM_BEACON_INTERVAL, .listen_interval = WMI_VDEV_PARAM_LISTEN_INTERVAL, .multicast_rate = WMI_VDEV_PARAM_MULTICAST_RATE, .mgmt_tx_rate = WMI_VDEV_PARAM_MGMT_TX_RATE, .slot_time = WMI_VDEV_PARAM_SLOT_TIME, .preamble = WMI_VDEV_PARAM_PREAMBLE, .swba_time = WMI_VDEV_PARAM_SWBA_TIME, .wmi_vdev_stats_update_period = WMI_VDEV_STATS_UPDATE_PERIOD, .wmi_vdev_pwrsave_ageout_time = WMI_VDEV_PWRSAVE_AGEOUT_TIME, .wmi_vdev_host_swba_interval = WMI_VDEV_HOST_SWBA_INTERVAL, .dtim_period = WMI_VDEV_PARAM_DTIM_PERIOD, .wmi_vdev_oc_scheduler_air_time_limit = WMI_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT, .wds = WMI_VDEV_PARAM_WDS, .atim_window = WMI_VDEV_PARAM_ATIM_WINDOW, .bmiss_count_max = WMI_VDEV_PARAM_BMISS_COUNT_MAX, .bmiss_first_bcnt = WMI_VDEV_PARAM_BMISS_FIRST_BCNT, .bmiss_final_bcnt = WMI_VDEV_PARAM_BMISS_FINAL_BCNT, .feature_wmm = WMI_VDEV_PARAM_FEATURE_WMM, .chwidth = WMI_VDEV_PARAM_CHWIDTH, .chextoffset = WMI_VDEV_PARAM_CHEXTOFFSET, .disable_htprotection = WMI_VDEV_PARAM_DISABLE_HTPROTECTION, .sta_quickkickout = WMI_VDEV_PARAM_STA_QUICKKICKOUT, .mgmt_rate = WMI_VDEV_PARAM_MGMT_RATE, .protection_mode = WMI_VDEV_PARAM_PROTECTION_MODE, .fixed_rate = WMI_VDEV_PARAM_FIXED_RATE, .sgi = WMI_VDEV_PARAM_SGI, .ldpc = WMI_VDEV_PARAM_LDPC, .tx_stbc = WMI_VDEV_PARAM_TX_STBC, .rx_stbc = WMI_VDEV_PARAM_RX_STBC, .intra_bss_fwd = WMI_VDEV_PARAM_INTRA_BSS_FWD, .def_keyid = WMI_VDEV_PARAM_DEF_KEYID, .nss = WMI_VDEV_PARAM_NSS, .bcast_data_rate = WMI_VDEV_PARAM_BCAST_DATA_RATE, .mcast_data_rate = WMI_VDEV_PARAM_MCAST_DATA_RATE, .mcast_indicate = WMI_VDEV_PARAM_MCAST_INDICATE, .dhcp_indicate = WMI_VDEV_PARAM_DHCP_INDICATE, .unknown_dest_indicate = WMI_VDEV_PARAM_UNKNOWN_DEST_INDICATE, .ap_keepalive_min_idle_inactive_time_secs = WMI_VDEV_PARAM_AP_KEEPALIVE_MIN_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_idle_inactive_time_secs = WMI_VDEV_PARAM_AP_KEEPALIVE_MAX_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_unresponsive_time_secs = WMI_VDEV_PARAM_AP_KEEPALIVE_MAX_UNRESPONSIVE_TIME_SECS, .ap_enable_nawds = WMI_VDEV_PARAM_AP_ENABLE_NAWDS, .mcast2ucast_set = WMI_VDEV_PARAM_UNSUPPORTED, .enable_rtscts = WMI_VDEV_PARAM_ENABLE_RTSCTS, .txbf = WMI_VDEV_PARAM_TXBF, .packet_powersave = WMI_VDEV_PARAM_PACKET_POWERSAVE, .drop_unencry = WMI_VDEV_PARAM_DROP_UNENCRY, .tx_encap_type = WMI_VDEV_PARAM_TX_ENCAP_TYPE, .ap_detect_out_of_sync_sleeping_sta_time_secs = WMI_VDEV_PARAM_UNSUPPORTED, .rc_num_retries = WMI_VDEV_PARAM_UNSUPPORTED, .cabq_maxdur = WMI_VDEV_PARAM_UNSUPPORTED, .mfptest_set = WMI_VDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_VDEV_PARAM_UNSUPPORTED, .vht_sgimask = WMI_VDEV_PARAM_UNSUPPORTED, .vht80_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_enable = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_tgt_bmiss_num = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_bmiss_sample_cycle = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_slop_step = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_init_slop = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_pause = WMI_VDEV_PARAM_UNSUPPORTED, .proxy_sta = WMI_VDEV_PARAM_UNSUPPORTED, .meru_vc = WMI_VDEV_PARAM_UNSUPPORTED, .rx_decap_type = WMI_VDEV_PARAM_UNSUPPORTED, .bw_nss_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .disable_4addr_src_lrn = WMI_VDEV_PARAM_UNSUPPORTED, .rtt_responder_role = WMI_VDEV_PARAM_UNSUPPORTED, }; /* 10.X WMI VDEV param map */ static struct wmi_vdev_param_map wmi_10x_vdev_param_map = { .rts_threshold = WMI_10X_VDEV_PARAM_RTS_THRESHOLD, .fragmentation_threshold = WMI_10X_VDEV_PARAM_FRAGMENTATION_THRESHOLD, .beacon_interval = WMI_10X_VDEV_PARAM_BEACON_INTERVAL, .listen_interval = WMI_10X_VDEV_PARAM_LISTEN_INTERVAL, .multicast_rate = WMI_10X_VDEV_PARAM_MULTICAST_RATE, .mgmt_tx_rate = WMI_10X_VDEV_PARAM_MGMT_TX_RATE, .slot_time = WMI_10X_VDEV_PARAM_SLOT_TIME, .preamble = WMI_10X_VDEV_PARAM_PREAMBLE, .swba_time = WMI_10X_VDEV_PARAM_SWBA_TIME, .wmi_vdev_stats_update_period = WMI_10X_VDEV_STATS_UPDATE_PERIOD, .wmi_vdev_pwrsave_ageout_time = WMI_10X_VDEV_PWRSAVE_AGEOUT_TIME, .wmi_vdev_host_swba_interval = WMI_10X_VDEV_HOST_SWBA_INTERVAL, .dtim_period = WMI_10X_VDEV_PARAM_DTIM_PERIOD, .wmi_vdev_oc_scheduler_air_time_limit = WMI_10X_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT, .wds = WMI_10X_VDEV_PARAM_WDS, .atim_window = WMI_10X_VDEV_PARAM_ATIM_WINDOW, .bmiss_count_max = WMI_10X_VDEV_PARAM_BMISS_COUNT_MAX, .bmiss_first_bcnt = WMI_VDEV_PARAM_UNSUPPORTED, .bmiss_final_bcnt = WMI_VDEV_PARAM_UNSUPPORTED, .feature_wmm = WMI_10X_VDEV_PARAM_FEATURE_WMM, .chwidth = WMI_10X_VDEV_PARAM_CHWIDTH, .chextoffset = WMI_10X_VDEV_PARAM_CHEXTOFFSET, .disable_htprotection = WMI_10X_VDEV_PARAM_DISABLE_HTPROTECTION, .sta_quickkickout = WMI_10X_VDEV_PARAM_STA_QUICKKICKOUT, .mgmt_rate = WMI_10X_VDEV_PARAM_MGMT_RATE, .protection_mode = WMI_10X_VDEV_PARAM_PROTECTION_MODE, .fixed_rate = WMI_10X_VDEV_PARAM_FIXED_RATE, .sgi = WMI_10X_VDEV_PARAM_SGI, .ldpc = WMI_10X_VDEV_PARAM_LDPC, .tx_stbc = WMI_10X_VDEV_PARAM_TX_STBC, .rx_stbc = WMI_10X_VDEV_PARAM_RX_STBC, .intra_bss_fwd = WMI_10X_VDEV_PARAM_INTRA_BSS_FWD, .def_keyid = WMI_10X_VDEV_PARAM_DEF_KEYID, .nss = WMI_10X_VDEV_PARAM_NSS, .bcast_data_rate = WMI_10X_VDEV_PARAM_BCAST_DATA_RATE, .mcast_data_rate = WMI_10X_VDEV_PARAM_MCAST_DATA_RATE, .mcast_indicate = WMI_10X_VDEV_PARAM_MCAST_INDICATE, .dhcp_indicate = WMI_10X_VDEV_PARAM_DHCP_INDICATE, .unknown_dest_indicate = WMI_10X_VDEV_PARAM_UNKNOWN_DEST_INDICATE, .ap_keepalive_min_idle_inactive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MIN_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_idle_inactive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MAX_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_unresponsive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MAX_UNRESPONSIVE_TIME_SECS, .ap_enable_nawds = WMI_10X_VDEV_PARAM_AP_ENABLE_NAWDS, .mcast2ucast_set = WMI_10X_VDEV_PARAM_MCAST2UCAST_SET, .enable_rtscts = WMI_10X_VDEV_PARAM_ENABLE_RTSCTS, .txbf = WMI_VDEV_PARAM_UNSUPPORTED, .packet_powersave = WMI_VDEV_PARAM_UNSUPPORTED, .drop_unencry = WMI_VDEV_PARAM_UNSUPPORTED, .tx_encap_type = WMI_VDEV_PARAM_UNSUPPORTED, .ap_detect_out_of_sync_sleeping_sta_time_secs = WMI_10X_VDEV_PARAM_AP_DETECT_OUT_OF_SYNC_SLEEPING_STA_TIME_SECS, .rc_num_retries = WMI_VDEV_PARAM_UNSUPPORTED, .cabq_maxdur = WMI_VDEV_PARAM_UNSUPPORTED, .mfptest_set = WMI_VDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_VDEV_PARAM_UNSUPPORTED, .vht_sgimask = WMI_VDEV_PARAM_UNSUPPORTED, .vht80_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_enable = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_tgt_bmiss_num = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_bmiss_sample_cycle = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_slop_step = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_init_slop = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_pause = WMI_VDEV_PARAM_UNSUPPORTED, .proxy_sta = WMI_VDEV_PARAM_UNSUPPORTED, .meru_vc = WMI_VDEV_PARAM_UNSUPPORTED, .rx_decap_type = WMI_VDEV_PARAM_UNSUPPORTED, .bw_nss_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .disable_4addr_src_lrn = WMI_VDEV_PARAM_UNSUPPORTED, .rtt_responder_role = WMI_VDEV_PARAM_UNSUPPORTED, }; static struct wmi_vdev_param_map wmi_10_2_4_vdev_param_map = { .rts_threshold = WMI_10X_VDEV_PARAM_RTS_THRESHOLD, .fragmentation_threshold = WMI_10X_VDEV_PARAM_FRAGMENTATION_THRESHOLD, .beacon_interval = WMI_10X_VDEV_PARAM_BEACON_INTERVAL, .listen_interval = WMI_10X_VDEV_PARAM_LISTEN_INTERVAL, .multicast_rate = WMI_10X_VDEV_PARAM_MULTICAST_RATE, .mgmt_tx_rate = WMI_10X_VDEV_PARAM_MGMT_TX_RATE, .slot_time = WMI_10X_VDEV_PARAM_SLOT_TIME, .preamble = WMI_10X_VDEV_PARAM_PREAMBLE, .swba_time = WMI_10X_VDEV_PARAM_SWBA_TIME, .wmi_vdev_stats_update_period = WMI_10X_VDEV_STATS_UPDATE_PERIOD, .wmi_vdev_pwrsave_ageout_time = WMI_10X_VDEV_PWRSAVE_AGEOUT_TIME, .wmi_vdev_host_swba_interval = WMI_10X_VDEV_HOST_SWBA_INTERVAL, .dtim_period = WMI_10X_VDEV_PARAM_DTIM_PERIOD, .wmi_vdev_oc_scheduler_air_time_limit = WMI_10X_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT, .wds = WMI_10X_VDEV_PARAM_WDS, .atim_window = WMI_10X_VDEV_PARAM_ATIM_WINDOW, .bmiss_count_max = WMI_10X_VDEV_PARAM_BMISS_COUNT_MAX, .bmiss_first_bcnt = WMI_VDEV_PARAM_UNSUPPORTED, .bmiss_final_bcnt = WMI_VDEV_PARAM_UNSUPPORTED, .feature_wmm = WMI_10X_VDEV_PARAM_FEATURE_WMM, .chwidth = WMI_10X_VDEV_PARAM_CHWIDTH, .chextoffset = WMI_10X_VDEV_PARAM_CHEXTOFFSET, .disable_htprotection = WMI_10X_VDEV_PARAM_DISABLE_HTPROTECTION, .sta_quickkickout = WMI_10X_VDEV_PARAM_STA_QUICKKICKOUT, .mgmt_rate = WMI_10X_VDEV_PARAM_MGMT_RATE, .protection_mode = WMI_10X_VDEV_PARAM_PROTECTION_MODE, .fixed_rate = WMI_10X_VDEV_PARAM_FIXED_RATE, .sgi = WMI_10X_VDEV_PARAM_SGI, .ldpc = WMI_10X_VDEV_PARAM_LDPC, .tx_stbc = WMI_10X_VDEV_PARAM_TX_STBC, .rx_stbc = WMI_10X_VDEV_PARAM_RX_STBC, .intra_bss_fwd = WMI_10X_VDEV_PARAM_INTRA_BSS_FWD, .def_keyid = WMI_10X_VDEV_PARAM_DEF_KEYID, .nss = WMI_10X_VDEV_PARAM_NSS, .bcast_data_rate = WMI_10X_VDEV_PARAM_BCAST_DATA_RATE, .mcast_data_rate = WMI_10X_VDEV_PARAM_MCAST_DATA_RATE, .mcast_indicate = WMI_10X_VDEV_PARAM_MCAST_INDICATE, .dhcp_indicate = WMI_10X_VDEV_PARAM_DHCP_INDICATE, .unknown_dest_indicate = WMI_10X_VDEV_PARAM_UNKNOWN_DEST_INDICATE, .ap_keepalive_min_idle_inactive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MIN_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_idle_inactive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MAX_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_unresponsive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MAX_UNRESPONSIVE_TIME_SECS, .ap_enable_nawds = WMI_10X_VDEV_PARAM_AP_ENABLE_NAWDS, .mcast2ucast_set = WMI_10X_VDEV_PARAM_MCAST2UCAST_SET, .enable_rtscts = WMI_10X_VDEV_PARAM_ENABLE_RTSCTS, .txbf = WMI_VDEV_PARAM_UNSUPPORTED, .packet_powersave = WMI_VDEV_PARAM_UNSUPPORTED, .drop_unencry = WMI_VDEV_PARAM_UNSUPPORTED, .tx_encap_type = WMI_VDEV_PARAM_UNSUPPORTED, .ap_detect_out_of_sync_sleeping_sta_time_secs = WMI_10X_VDEV_PARAM_AP_DETECT_OUT_OF_SYNC_SLEEPING_STA_TIME_SECS, .rc_num_retries = WMI_VDEV_PARAM_UNSUPPORTED, .cabq_maxdur = WMI_VDEV_PARAM_UNSUPPORTED, .mfptest_set = WMI_VDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_VDEV_PARAM_UNSUPPORTED, .vht_sgimask = WMI_VDEV_PARAM_UNSUPPORTED, .vht80_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_enable = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_tgt_bmiss_num = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_bmiss_sample_cycle = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_slop_step = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_init_slop = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_pause = WMI_VDEV_PARAM_UNSUPPORTED, .proxy_sta = WMI_VDEV_PARAM_UNSUPPORTED, .meru_vc = WMI_VDEV_PARAM_UNSUPPORTED, .rx_decap_type = WMI_VDEV_PARAM_UNSUPPORTED, .bw_nss_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .disable_4addr_src_lrn = WMI_VDEV_PARAM_UNSUPPORTED, .rtt_responder_role = WMI_VDEV_PARAM_UNSUPPORTED, }; static struct wmi_vdev_param_map wmi_10_4_vdev_param_map = { .rts_threshold = WMI_10_4_VDEV_PARAM_RTS_THRESHOLD, .fragmentation_threshold = WMI_10_4_VDEV_PARAM_FRAGMENTATION_THRESHOLD, .beacon_interval = WMI_10_4_VDEV_PARAM_BEACON_INTERVAL, .listen_interval = WMI_10_4_VDEV_PARAM_LISTEN_INTERVAL, .multicast_rate = WMI_10_4_VDEV_PARAM_MULTICAST_RATE, .mgmt_tx_rate = WMI_10_4_VDEV_PARAM_MGMT_TX_RATE, .slot_time = WMI_10_4_VDEV_PARAM_SLOT_TIME, .preamble = WMI_10_4_VDEV_PARAM_PREAMBLE, .swba_time = WMI_10_4_VDEV_PARAM_SWBA_TIME, .wmi_vdev_stats_update_period = WMI_10_4_VDEV_STATS_UPDATE_PERIOD, .wmi_vdev_pwrsave_ageout_time = WMI_10_4_VDEV_PWRSAVE_AGEOUT_TIME, .wmi_vdev_host_swba_interval = WMI_10_4_VDEV_HOST_SWBA_INTERVAL, .dtim_period = WMI_10_4_VDEV_PARAM_DTIM_PERIOD, .wmi_vdev_oc_scheduler_air_time_limit = WMI_10_4_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT, .wds = WMI_10_4_VDEV_PARAM_WDS, .atim_window = WMI_10_4_VDEV_PARAM_ATIM_WINDOW, .bmiss_count_max = WMI_10_4_VDEV_PARAM_BMISS_COUNT_MAX, .bmiss_first_bcnt = WMI_10_4_VDEV_PARAM_BMISS_FIRST_BCNT, .bmiss_final_bcnt = WMI_10_4_VDEV_PARAM_BMISS_FINAL_BCNT, .feature_wmm = WMI_10_4_VDEV_PARAM_FEATURE_WMM, .chwidth = WMI_10_4_VDEV_PARAM_CHWIDTH, .chextoffset = WMI_10_4_VDEV_PARAM_CHEXTOFFSET, .disable_htprotection = WMI_10_4_VDEV_PARAM_DISABLE_HTPROTECTION, .sta_quickkickout = WMI_10_4_VDEV_PARAM_STA_QUICKKICKOUT, .mgmt_rate = WMI_10_4_VDEV_PARAM_MGMT_RATE, .protection_mode = WMI_10_4_VDEV_PARAM_PROTECTION_MODE, .fixed_rate = WMI_10_4_VDEV_PARAM_FIXED_RATE, .sgi = WMI_10_4_VDEV_PARAM_SGI, .ldpc = WMI_10_4_VDEV_PARAM_LDPC, .tx_stbc = WMI_10_4_VDEV_PARAM_TX_STBC, .rx_stbc = WMI_10_4_VDEV_PARAM_RX_STBC, .intra_bss_fwd = WMI_10_4_VDEV_PARAM_INTRA_BSS_FWD, .def_keyid = WMI_10_4_VDEV_PARAM_DEF_KEYID, .nss = WMI_10_4_VDEV_PARAM_NSS, .bcast_data_rate = WMI_10_4_VDEV_PARAM_BCAST_DATA_RATE, .mcast_data_rate = WMI_10_4_VDEV_PARAM_MCAST_DATA_RATE, .mcast_indicate = WMI_10_4_VDEV_PARAM_MCAST_INDICATE, .dhcp_indicate = WMI_10_4_VDEV_PARAM_DHCP_INDICATE, .unknown_dest_indicate = WMI_10_4_VDEV_PARAM_UNKNOWN_DEST_INDICATE, .ap_keepalive_min_idle_inactive_time_secs = WMI_10_4_VDEV_PARAM_AP_KEEPALIVE_MIN_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_idle_inactive_time_secs = WMI_10_4_VDEV_PARAM_AP_KEEPALIVE_MAX_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_unresponsive_time_secs = WMI_10_4_VDEV_PARAM_AP_KEEPALIVE_MAX_UNRESPONSIVE_TIME_SECS, .ap_enable_nawds = WMI_10_4_VDEV_PARAM_AP_ENABLE_NAWDS, .mcast2ucast_set = WMI_10_4_VDEV_PARAM_MCAST2UCAST_SET, .enable_rtscts = WMI_10_4_VDEV_PARAM_ENABLE_RTSCTS, .txbf = WMI_10_4_VDEV_PARAM_TXBF, .packet_powersave = WMI_10_4_VDEV_PARAM_PACKET_POWERSAVE, .drop_unencry = WMI_10_4_VDEV_PARAM_DROP_UNENCRY, .tx_encap_type = WMI_10_4_VDEV_PARAM_TX_ENCAP_TYPE, .ap_detect_out_of_sync_sleeping_sta_time_secs = WMI_10_4_VDEV_PARAM_AP_DETECT_OUT_OF_SYNC_SLEEPING_STA_TIME_SECS, .rc_num_retries = WMI_10_4_VDEV_PARAM_RC_NUM_RETRIES, .cabq_maxdur = WMI_10_4_VDEV_PARAM_CABQ_MAXDUR, .mfptest_set = WMI_10_4_VDEV_PARAM_MFPTEST_SET, .rts_fixed_rate = WMI_10_4_VDEV_PARAM_RTS_FIXED_RATE, .vht_sgimask = WMI_10_4_VDEV_PARAM_VHT_SGIMASK, .vht80_ratemask = WMI_10_4_VDEV_PARAM_VHT80_RATEMASK, .early_rx_adjust_enable = WMI_10_4_VDEV_PARAM_EARLY_RX_ADJUST_ENABLE, .early_rx_tgt_bmiss_num = WMI_10_4_VDEV_PARAM_EARLY_RX_TGT_BMISS_NUM, .early_rx_bmiss_sample_cycle = WMI_10_4_VDEV_PARAM_EARLY_RX_BMISS_SAMPLE_CYCLE, .early_rx_slop_step = WMI_10_4_VDEV_PARAM_EARLY_RX_SLOP_STEP, .early_rx_init_slop = WMI_10_4_VDEV_PARAM_EARLY_RX_INIT_SLOP, .early_rx_adjust_pause = WMI_10_4_VDEV_PARAM_EARLY_RX_ADJUST_PAUSE, .proxy_sta = WMI_10_4_VDEV_PARAM_PROXY_STA, .meru_vc = WMI_10_4_VDEV_PARAM_MERU_VC, .rx_decap_type = WMI_10_4_VDEV_PARAM_RX_DECAP_TYPE, .bw_nss_ratemask = WMI_10_4_VDEV_PARAM_BW_NSS_RATEMASK, .inc_tsf = WMI_10_4_VDEV_PARAM_TSF_INCREMENT, .dec_tsf = WMI_10_4_VDEV_PARAM_TSF_DECREMENT, .disable_4addr_src_lrn = WMI_10_4_VDEV_PARAM_DISABLE_4_ADDR_SRC_LRN, .rtt_responder_role = WMI_10_4_VDEV_PARAM_ENABLE_DISABLE_RTT_RESPONDER_ROLE, }; static struct wmi_pdev_param_map wmi_pdev_param_map = { .tx_chain_mask = WMI_PDEV_PARAM_TX_CHAIN_MASK, .rx_chain_mask = WMI_PDEV_PARAM_RX_CHAIN_MASK, .txpower_limit2g = WMI_PDEV_PARAM_TXPOWER_LIMIT2G, .txpower_limit5g = WMI_PDEV_PARAM_TXPOWER_LIMIT5G, .txpower_scale = WMI_PDEV_PARAM_TXPOWER_SCALE, .beacon_gen_mode = WMI_PDEV_PARAM_BEACON_GEN_MODE, .beacon_tx_mode = WMI_PDEV_PARAM_BEACON_TX_MODE, .resmgr_offchan_mode = WMI_PDEV_PARAM_RESMGR_OFFCHAN_MODE, .protection_mode = WMI_PDEV_PARAM_PROTECTION_MODE, .dynamic_bw = WMI_PDEV_PARAM_DYNAMIC_BW, .non_agg_sw_retry_th = WMI_PDEV_PARAM_NON_AGG_SW_RETRY_TH, .agg_sw_retry_th = WMI_PDEV_PARAM_AGG_SW_RETRY_TH, .sta_kickout_th = WMI_PDEV_PARAM_STA_KICKOUT_TH, .ac_aggrsize_scaling = WMI_PDEV_PARAM_AC_AGGRSIZE_SCALING, .ltr_enable = WMI_PDEV_PARAM_LTR_ENABLE, .ltr_ac_latency_be = WMI_PDEV_PARAM_LTR_AC_LATENCY_BE, .ltr_ac_latency_bk = WMI_PDEV_PARAM_LTR_AC_LATENCY_BK, .ltr_ac_latency_vi = WMI_PDEV_PARAM_LTR_AC_LATENCY_VI, .ltr_ac_latency_vo = WMI_PDEV_PARAM_LTR_AC_LATENCY_VO, .ltr_ac_latency_timeout = WMI_PDEV_PARAM_LTR_AC_LATENCY_TIMEOUT, .ltr_sleep_override = WMI_PDEV_PARAM_LTR_SLEEP_OVERRIDE, .ltr_rx_override = WMI_PDEV_PARAM_LTR_RX_OVERRIDE, .ltr_tx_activity_timeout = WMI_PDEV_PARAM_LTR_TX_ACTIVITY_TIMEOUT, .l1ss_enable = WMI_PDEV_PARAM_L1SS_ENABLE, .dsleep_enable = WMI_PDEV_PARAM_DSLEEP_ENABLE, .pcielp_txbuf_flush = WMI_PDEV_PARAM_PCIELP_TXBUF_FLUSH, .pcielp_txbuf_watermark = WMI_PDEV_PARAM_PCIELP_TXBUF_TMO_EN, .pcielp_txbuf_tmo_en = WMI_PDEV_PARAM_PCIELP_TXBUF_TMO_EN, .pcielp_txbuf_tmo_value = WMI_PDEV_PARAM_PCIELP_TXBUF_TMO_VALUE, .pdev_stats_update_period = WMI_PDEV_PARAM_PDEV_STATS_UPDATE_PERIOD, .vdev_stats_update_period = WMI_PDEV_PARAM_VDEV_STATS_UPDATE_PERIOD, .peer_stats_update_period = WMI_PDEV_PARAM_PEER_STATS_UPDATE_PERIOD, .bcnflt_stats_update_period = WMI_PDEV_PARAM_BCNFLT_STATS_UPDATE_PERIOD, .pmf_qos = WMI_PDEV_PARAM_PMF_QOS, .arp_ac_override = WMI_PDEV_PARAM_ARP_AC_OVERRIDE, .dcs = WMI_PDEV_PARAM_DCS, .ani_enable = WMI_PDEV_PARAM_ANI_ENABLE, .ani_poll_period = WMI_PDEV_PARAM_ANI_POLL_PERIOD, .ani_listen_period = WMI_PDEV_PARAM_ANI_LISTEN_PERIOD, .ani_ofdm_level = WMI_PDEV_PARAM_ANI_OFDM_LEVEL, .ani_cck_level = WMI_PDEV_PARAM_ANI_CCK_LEVEL, .dyntxchain = WMI_PDEV_PARAM_DYNTXCHAIN, .proxy_sta = WMI_PDEV_PARAM_PROXY_STA, .idle_ps_config = WMI_PDEV_PARAM_IDLE_PS_CONFIG, .power_gating_sleep = WMI_PDEV_PARAM_POWER_GATING_SLEEP, .fast_channel_reset = WMI_PDEV_PARAM_UNSUPPORTED, .burst_dur = WMI_PDEV_PARAM_UNSUPPORTED, .burst_enable = WMI_PDEV_PARAM_UNSUPPORTED, .cal_period = WMI_PDEV_PARAM_UNSUPPORTED, .aggr_burst = WMI_PDEV_PARAM_UNSUPPORTED, .rx_decap_mode = WMI_PDEV_PARAM_UNSUPPORTED, .smart_antenna_default_antenna = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_override = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_tid = WMI_PDEV_PARAM_UNSUPPORTED, .antenna_gain = WMI_PDEV_PARAM_UNSUPPORTED, .rx_filter = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_to_ucast_tid = WMI_PDEV_PARAM_UNSUPPORTED, .proxy_sta_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .remove_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .peer_sta_ps_statechg_enable = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_ac_override = WMI_PDEV_PARAM_UNSUPPORTED, .block_interbss = WMI_PDEV_PARAM_UNSUPPORTED, .set_disable_reset_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_msdu_ttl_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_ppdu_duration_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .txbf_sound_period_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_promisc_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_burst_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .en_stats = WMI_PDEV_PARAM_UNSUPPORTED, .mu_group_policy = WMI_PDEV_PARAM_UNSUPPORTED, .noise_detection = WMI_PDEV_PARAM_UNSUPPORTED, .noise_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .dpd_enable = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_bcast_echo = WMI_PDEV_PARAM_UNSUPPORTED, .atf_strict_sch = WMI_PDEV_PARAM_UNSUPPORTED, .atf_sched_duration = WMI_PDEV_PARAM_UNSUPPORTED, .ant_plzn = WMI_PDEV_PARAM_UNSUPPORTED, .mgmt_retry_limit = WMI_PDEV_PARAM_UNSUPPORTED, .sensitivity_level = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_2g = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_5g = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_amsdu = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_ampdu = WMI_PDEV_PARAM_UNSUPPORTED, .cca_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_PDEV_PARAM_UNSUPPORTED, .pdev_reset = WMI_PDEV_PARAM_UNSUPPORTED, .wapi_mbssid_offset = WMI_PDEV_PARAM_UNSUPPORTED, .arp_srcaddr = WMI_PDEV_PARAM_UNSUPPORTED, .arp_dstaddr = WMI_PDEV_PARAM_UNSUPPORTED, .enable_btcoex = WMI_PDEV_PARAM_UNSUPPORTED, }; static struct wmi_pdev_param_map wmi_10x_pdev_param_map = { .tx_chain_mask = WMI_10X_PDEV_PARAM_TX_CHAIN_MASK, .rx_chain_mask = WMI_10X_PDEV_PARAM_RX_CHAIN_MASK, .txpower_limit2g = WMI_10X_PDEV_PARAM_TXPOWER_LIMIT2G, .txpower_limit5g = WMI_10X_PDEV_PARAM_TXPOWER_LIMIT5G, .txpower_scale = WMI_10X_PDEV_PARAM_TXPOWER_SCALE, .beacon_gen_mode = WMI_10X_PDEV_PARAM_BEACON_GEN_MODE, .beacon_tx_mode = WMI_10X_PDEV_PARAM_BEACON_TX_MODE, .resmgr_offchan_mode = WMI_10X_PDEV_PARAM_RESMGR_OFFCHAN_MODE, .protection_mode = WMI_10X_PDEV_PARAM_PROTECTION_MODE, .dynamic_bw = WMI_10X_PDEV_PARAM_DYNAMIC_BW, .non_agg_sw_retry_th = WMI_10X_PDEV_PARAM_NON_AGG_SW_RETRY_TH, .agg_sw_retry_th = WMI_10X_PDEV_PARAM_AGG_SW_RETRY_TH, .sta_kickout_th = WMI_10X_PDEV_PARAM_STA_KICKOUT_TH, .ac_aggrsize_scaling = WMI_10X_PDEV_PARAM_AC_AGGRSIZE_SCALING, .ltr_enable = WMI_10X_PDEV_PARAM_LTR_ENABLE, .ltr_ac_latency_be = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_BE, .ltr_ac_latency_bk = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_BK, .ltr_ac_latency_vi = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_VI, .ltr_ac_latency_vo = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_VO, .ltr_ac_latency_timeout = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_TIMEOUT, .ltr_sleep_override = WMI_10X_PDEV_PARAM_LTR_SLEEP_OVERRIDE, .ltr_rx_override = WMI_10X_PDEV_PARAM_LTR_RX_OVERRIDE, .ltr_tx_activity_timeout = WMI_10X_PDEV_PARAM_LTR_TX_ACTIVITY_TIMEOUT, .l1ss_enable = WMI_10X_PDEV_PARAM_L1SS_ENABLE, .dsleep_enable = WMI_10X_PDEV_PARAM_DSLEEP_ENABLE, .pcielp_txbuf_flush = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_watermark = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_tmo_en = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_tmo_value = WMI_PDEV_PARAM_UNSUPPORTED, .pdev_stats_update_period = WMI_10X_PDEV_PARAM_PDEV_STATS_UPDATE_PERIOD, .vdev_stats_update_period = WMI_10X_PDEV_PARAM_VDEV_STATS_UPDATE_PERIOD, .peer_stats_update_period = WMI_10X_PDEV_PARAM_PEER_STATS_UPDATE_PERIOD, .bcnflt_stats_update_period = WMI_10X_PDEV_PARAM_BCNFLT_STATS_UPDATE_PERIOD, .pmf_qos = WMI_10X_PDEV_PARAM_PMF_QOS, .arp_ac_override = WMI_10X_PDEV_PARAM_ARPDHCP_AC_OVERRIDE, .dcs = WMI_10X_PDEV_PARAM_DCS, .ani_enable = WMI_10X_PDEV_PARAM_ANI_ENABLE, .ani_poll_period = WMI_10X_PDEV_PARAM_ANI_POLL_PERIOD, .ani_listen_period = WMI_10X_PDEV_PARAM_ANI_LISTEN_PERIOD, .ani_ofdm_level = WMI_10X_PDEV_PARAM_ANI_OFDM_LEVEL, .ani_cck_level = WMI_10X_PDEV_PARAM_ANI_CCK_LEVEL, .dyntxchain = WMI_10X_PDEV_PARAM_DYNTXCHAIN, .proxy_sta = WMI_PDEV_PARAM_UNSUPPORTED, .idle_ps_config = WMI_PDEV_PARAM_UNSUPPORTED, .power_gating_sleep = WMI_PDEV_PARAM_UNSUPPORTED, .fast_channel_reset = WMI_10X_PDEV_PARAM_FAST_CHANNEL_RESET, .burst_dur = WMI_10X_PDEV_PARAM_BURST_DUR, .burst_enable = WMI_10X_PDEV_PARAM_BURST_ENABLE, .cal_period = WMI_10X_PDEV_PARAM_CAL_PERIOD, .aggr_burst = WMI_PDEV_PARAM_UNSUPPORTED, .rx_decap_mode = WMI_PDEV_PARAM_UNSUPPORTED, .smart_antenna_default_antenna = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_override = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_tid = WMI_PDEV_PARAM_UNSUPPORTED, .antenna_gain = WMI_PDEV_PARAM_UNSUPPORTED, .rx_filter = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_to_ucast_tid = WMI_PDEV_PARAM_UNSUPPORTED, .proxy_sta_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .remove_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .peer_sta_ps_statechg_enable = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_ac_override = WMI_PDEV_PARAM_UNSUPPORTED, .block_interbss = WMI_PDEV_PARAM_UNSUPPORTED, .set_disable_reset_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_msdu_ttl_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_ppdu_duration_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .txbf_sound_period_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_promisc_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_burst_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .en_stats = WMI_PDEV_PARAM_UNSUPPORTED, .mu_group_policy = WMI_PDEV_PARAM_UNSUPPORTED, .noise_detection = WMI_PDEV_PARAM_UNSUPPORTED, .noise_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .dpd_enable = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_bcast_echo = WMI_PDEV_PARAM_UNSUPPORTED, .atf_strict_sch = WMI_PDEV_PARAM_UNSUPPORTED, .atf_sched_duration = WMI_PDEV_PARAM_UNSUPPORTED, .ant_plzn = WMI_PDEV_PARAM_UNSUPPORTED, .mgmt_retry_limit = WMI_PDEV_PARAM_UNSUPPORTED, .sensitivity_level = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_2g = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_5g = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_amsdu = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_ampdu = WMI_PDEV_PARAM_UNSUPPORTED, .cca_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_PDEV_PARAM_UNSUPPORTED, .pdev_reset = WMI_PDEV_PARAM_UNSUPPORTED, .wapi_mbssid_offset = WMI_PDEV_PARAM_UNSUPPORTED, .arp_srcaddr = WMI_PDEV_PARAM_UNSUPPORTED, .arp_dstaddr = WMI_PDEV_PARAM_UNSUPPORTED, .enable_btcoex = WMI_PDEV_PARAM_UNSUPPORTED, }; static struct wmi_pdev_param_map wmi_10_2_4_pdev_param_map = { .tx_chain_mask = WMI_10X_PDEV_PARAM_TX_CHAIN_MASK, .rx_chain_mask = WMI_10X_PDEV_PARAM_RX_CHAIN_MASK, .txpower_limit2g = WMI_10X_PDEV_PARAM_TXPOWER_LIMIT2G, .txpower_limit5g = WMI_10X_PDEV_PARAM_TXPOWER_LIMIT5G, .txpower_scale = WMI_10X_PDEV_PARAM_TXPOWER_SCALE, .beacon_gen_mode = WMI_10X_PDEV_PARAM_BEACON_GEN_MODE, .beacon_tx_mode = WMI_10X_PDEV_PARAM_BEACON_TX_MODE, .resmgr_offchan_mode = WMI_10X_PDEV_PARAM_RESMGR_OFFCHAN_MODE, .protection_mode = WMI_10X_PDEV_PARAM_PROTECTION_MODE, .dynamic_bw = WMI_10X_PDEV_PARAM_DYNAMIC_BW, .non_agg_sw_retry_th = WMI_10X_PDEV_PARAM_NON_AGG_SW_RETRY_TH, .agg_sw_retry_th = WMI_10X_PDEV_PARAM_AGG_SW_RETRY_TH, .sta_kickout_th = WMI_10X_PDEV_PARAM_STA_KICKOUT_TH, .ac_aggrsize_scaling = WMI_10X_PDEV_PARAM_AC_AGGRSIZE_SCALING, .ltr_enable = WMI_10X_PDEV_PARAM_LTR_ENABLE, .ltr_ac_latency_be = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_BE, .ltr_ac_latency_bk = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_BK, .ltr_ac_latency_vi = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_VI, .ltr_ac_latency_vo = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_VO, .ltr_ac_latency_timeout = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_TIMEOUT, .ltr_sleep_override = WMI_10X_PDEV_PARAM_LTR_SLEEP_OVERRIDE, .ltr_rx_override = WMI_10X_PDEV_PARAM_LTR_RX_OVERRIDE, .ltr_tx_activity_timeout = WMI_10X_PDEV_PARAM_LTR_TX_ACTIVITY_TIMEOUT, .l1ss_enable = WMI_10X_PDEV_PARAM_L1SS_ENABLE, .dsleep_enable = WMI_10X_PDEV_PARAM_DSLEEP_ENABLE, .pcielp_txbuf_flush = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_watermark = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_tmo_en = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_tmo_value = WMI_PDEV_PARAM_UNSUPPORTED, .pdev_stats_update_period = WMI_10X_PDEV_PARAM_PDEV_STATS_UPDATE_PERIOD, .vdev_stats_update_period = WMI_10X_PDEV_PARAM_VDEV_STATS_UPDATE_PERIOD, .peer_stats_update_period = WMI_10X_PDEV_PARAM_PEER_STATS_UPDATE_PERIOD, .bcnflt_stats_update_period = WMI_10X_PDEV_PARAM_BCNFLT_STATS_UPDATE_PERIOD, .pmf_qos = WMI_10X_PDEV_PARAM_PMF_QOS, .arp_ac_override = WMI_10X_PDEV_PARAM_ARPDHCP_AC_OVERRIDE, .dcs = WMI_10X_PDEV_PARAM_DCS, .ani_enable = WMI_10X_PDEV_PARAM_ANI_ENABLE, .ani_poll_period = WMI_10X_PDEV_PARAM_ANI_POLL_PERIOD, .ani_listen_period = WMI_10X_PDEV_PARAM_ANI_LISTEN_PERIOD, .ani_ofdm_level = WMI_10X_PDEV_PARAM_ANI_OFDM_LEVEL, .ani_cck_level = WMI_10X_PDEV_PARAM_ANI_CCK_LEVEL, .dyntxchain = WMI_10X_PDEV_PARAM_DYNTXCHAIN, .proxy_sta = WMI_PDEV_PARAM_UNSUPPORTED, .idle_ps_config = WMI_PDEV_PARAM_UNSUPPORTED, .power_gating_sleep = WMI_PDEV_PARAM_UNSUPPORTED, .fast_channel_reset = WMI_10X_PDEV_PARAM_FAST_CHANNEL_RESET, .burst_dur = WMI_10X_PDEV_PARAM_BURST_DUR, .burst_enable = WMI_10X_PDEV_PARAM_BURST_ENABLE, .cal_period = WMI_10X_PDEV_PARAM_CAL_PERIOD, .aggr_burst = WMI_PDEV_PARAM_UNSUPPORTED, .rx_decap_mode = WMI_PDEV_PARAM_UNSUPPORTED, .smart_antenna_default_antenna = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_override = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_tid = WMI_PDEV_PARAM_UNSUPPORTED, .antenna_gain = WMI_PDEV_PARAM_UNSUPPORTED, .rx_filter = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_to_ucast_tid = WMI_PDEV_PARAM_UNSUPPORTED, .proxy_sta_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .remove_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .peer_sta_ps_statechg_enable = WMI_10X_PDEV_PARAM_PEER_STA_PS_STATECHG_ENABLE, .igmpmld_ac_override = WMI_PDEV_PARAM_UNSUPPORTED, .block_interbss = WMI_PDEV_PARAM_UNSUPPORTED, .set_disable_reset_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_msdu_ttl_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_ppdu_duration_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .txbf_sound_period_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_promisc_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_burst_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .en_stats = WMI_PDEV_PARAM_UNSUPPORTED, .mu_group_policy = WMI_PDEV_PARAM_UNSUPPORTED, .noise_detection = WMI_PDEV_PARAM_UNSUPPORTED, .noise_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .dpd_enable = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_bcast_echo = WMI_PDEV_PARAM_UNSUPPORTED, .atf_strict_sch = WMI_PDEV_PARAM_UNSUPPORTED, .atf_sched_duration = WMI_PDEV_PARAM_UNSUPPORTED, .ant_plzn = WMI_PDEV_PARAM_UNSUPPORTED, .mgmt_retry_limit = WMI_PDEV_PARAM_UNSUPPORTED, .sensitivity_level = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_2g = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_5g = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_amsdu = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_ampdu = WMI_PDEV_PARAM_UNSUPPORTED, .cca_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_PDEV_PARAM_UNSUPPORTED, .pdev_reset = WMI_10X_PDEV_PARAM_PDEV_RESET, .wapi_mbssid_offset = WMI_PDEV_PARAM_UNSUPPORTED, .arp_srcaddr = WMI_PDEV_PARAM_UNSUPPORTED, .arp_dstaddr = WMI_PDEV_PARAM_UNSUPPORTED, .enable_btcoex = WMI_PDEV_PARAM_UNSUPPORTED, }; /* firmware 10.2 specific mappings */ static struct wmi_cmd_map wmi_10_2_cmd_map = { .init_cmdid = WMI_10_2_INIT_CMDID, .start_scan_cmdid = WMI_10_2_START_SCAN_CMDID, .stop_scan_cmdid = WMI_10_2_STOP_SCAN_CMDID, .scan_chan_list_cmdid = WMI_10_2_SCAN_CHAN_LIST_CMDID, .scan_sch_prio_tbl_cmdid = WMI_CMD_UNSUPPORTED, .scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_regdomain_cmdid = WMI_10_2_PDEV_SET_REGDOMAIN_CMDID, .pdev_set_channel_cmdid = WMI_10_2_PDEV_SET_CHANNEL_CMDID, .pdev_set_param_cmdid = WMI_10_2_PDEV_SET_PARAM_CMDID, .pdev_pktlog_enable_cmdid = WMI_10_2_PDEV_PKTLOG_ENABLE_CMDID, .pdev_pktlog_disable_cmdid = WMI_10_2_PDEV_PKTLOG_DISABLE_CMDID, .pdev_set_wmm_params_cmdid = WMI_10_2_PDEV_SET_WMM_PARAMS_CMDID, .pdev_set_ht_cap_ie_cmdid = WMI_10_2_PDEV_SET_HT_CAP_IE_CMDID, .pdev_set_vht_cap_ie_cmdid = WMI_10_2_PDEV_SET_VHT_CAP_IE_CMDID, .pdev_set_quiet_mode_cmdid = WMI_10_2_PDEV_SET_QUIET_MODE_CMDID, .pdev_green_ap_ps_enable_cmdid = WMI_10_2_PDEV_GREEN_AP_PS_ENABLE_CMDID, .pdev_get_tpc_config_cmdid = WMI_10_2_PDEV_GET_TPC_CONFIG_CMDID, .pdev_set_base_macaddr_cmdid = WMI_10_2_PDEV_SET_BASE_MACADDR_CMDID, .vdev_create_cmdid = WMI_10_2_VDEV_CREATE_CMDID, .vdev_delete_cmdid = WMI_10_2_VDEV_DELETE_CMDID, .vdev_start_request_cmdid = WMI_10_2_VDEV_START_REQUEST_CMDID, .vdev_restart_request_cmdid = WMI_10_2_VDEV_RESTART_REQUEST_CMDID, .vdev_up_cmdid = WMI_10_2_VDEV_UP_CMDID, .vdev_stop_cmdid = WMI_10_2_VDEV_STOP_CMDID, .vdev_down_cmdid = WMI_10_2_VDEV_DOWN_CMDID, .vdev_set_param_cmdid = WMI_10_2_VDEV_SET_PARAM_CMDID, .vdev_install_key_cmdid = WMI_10_2_VDEV_INSTALL_KEY_CMDID, .peer_create_cmdid = WMI_10_2_PEER_CREATE_CMDID, .peer_delete_cmdid = WMI_10_2_PEER_DELETE_CMDID, .peer_flush_tids_cmdid = WMI_10_2_PEER_FLUSH_TIDS_CMDID, .peer_set_param_cmdid = WMI_10_2_PEER_SET_PARAM_CMDID, .peer_assoc_cmdid = WMI_10_2_PEER_ASSOC_CMDID, .peer_add_wds_entry_cmdid = WMI_10_2_PEER_ADD_WDS_ENTRY_CMDID, .peer_remove_wds_entry_cmdid = WMI_10_2_PEER_REMOVE_WDS_ENTRY_CMDID, .peer_mcast_group_cmdid = WMI_10_2_PEER_MCAST_GROUP_CMDID, .bcn_tx_cmdid = WMI_10_2_BCN_TX_CMDID, .pdev_send_bcn_cmdid = WMI_10_2_PDEV_SEND_BCN_CMDID, .bcn_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .bcn_filter_rx_cmdid = WMI_10_2_BCN_FILTER_RX_CMDID, .prb_req_filter_rx_cmdid = WMI_10_2_PRB_REQ_FILTER_RX_CMDID, .mgmt_tx_cmdid = WMI_10_2_MGMT_TX_CMDID, .prb_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .addba_clear_resp_cmdid = WMI_10_2_ADDBA_CLEAR_RESP_CMDID, .addba_send_cmdid = WMI_10_2_ADDBA_SEND_CMDID, .addba_status_cmdid = WMI_10_2_ADDBA_STATUS_CMDID, .delba_send_cmdid = WMI_10_2_DELBA_SEND_CMDID, .addba_set_resp_cmdid = WMI_10_2_ADDBA_SET_RESP_CMDID, .send_singleamsdu_cmdid = WMI_10_2_SEND_SINGLEAMSDU_CMDID, .sta_powersave_mode_cmdid = WMI_10_2_STA_POWERSAVE_MODE_CMDID, .sta_powersave_param_cmdid = WMI_10_2_STA_POWERSAVE_PARAM_CMDID, .sta_mimo_ps_mode_cmdid = WMI_10_2_STA_MIMO_PS_MODE_CMDID, .pdev_dfs_enable_cmdid = WMI_10_2_PDEV_DFS_ENABLE_CMDID, .pdev_dfs_disable_cmdid = WMI_10_2_PDEV_DFS_DISABLE_CMDID, .roam_scan_mode = WMI_10_2_ROAM_SCAN_MODE, .roam_scan_rssi_threshold = WMI_10_2_ROAM_SCAN_RSSI_THRESHOLD, .roam_scan_period = WMI_10_2_ROAM_SCAN_PERIOD, .roam_scan_rssi_change_threshold = WMI_10_2_ROAM_SCAN_RSSI_CHANGE_THRESHOLD, .roam_ap_profile = WMI_10_2_ROAM_AP_PROFILE, .ofl_scan_add_ap_profile = WMI_10_2_OFL_SCAN_ADD_AP_PROFILE, .ofl_scan_remove_ap_profile = WMI_10_2_OFL_SCAN_REMOVE_AP_PROFILE, .ofl_scan_period = WMI_10_2_OFL_SCAN_PERIOD, .p2p_dev_set_device_info = WMI_10_2_P2P_DEV_SET_DEVICE_INFO, .p2p_dev_set_discoverability = WMI_10_2_P2P_DEV_SET_DISCOVERABILITY, .p2p_go_set_beacon_ie = WMI_10_2_P2P_GO_SET_BEACON_IE, .p2p_go_set_probe_resp_ie = WMI_10_2_P2P_GO_SET_PROBE_RESP_IE, .p2p_set_vendor_ie_data_cmdid = WMI_CMD_UNSUPPORTED, .ap_ps_peer_param_cmdid = WMI_10_2_AP_PS_PEER_PARAM_CMDID, .ap_ps_peer_uapsd_coex_cmdid = WMI_CMD_UNSUPPORTED, .peer_rate_retry_sched_cmdid = WMI_10_2_PEER_RATE_RETRY_SCHED_CMDID, .wlan_profile_trigger_cmdid = WMI_10_2_WLAN_PROFILE_TRIGGER_CMDID, .wlan_profile_set_hist_intvl_cmdid = WMI_10_2_WLAN_PROFILE_SET_HIST_INTVL_CMDID, .wlan_profile_get_profile_data_cmdid = WMI_10_2_WLAN_PROFILE_GET_PROFILE_DATA_CMDID, .wlan_profile_enable_profile_id_cmdid = WMI_10_2_WLAN_PROFILE_ENABLE_PROFILE_ID_CMDID, .wlan_profile_list_profile_id_cmdid = WMI_10_2_WLAN_PROFILE_LIST_PROFILE_ID_CMDID, .pdev_suspend_cmdid = WMI_10_2_PDEV_SUSPEND_CMDID, .pdev_resume_cmdid = WMI_10_2_PDEV_RESUME_CMDID, .add_bcn_filter_cmdid = WMI_10_2_ADD_BCN_FILTER_CMDID, .rmv_bcn_filter_cmdid = WMI_10_2_RMV_BCN_FILTER_CMDID, .wow_add_wake_pattern_cmdid = WMI_10_2_WOW_ADD_WAKE_PATTERN_CMDID, .wow_del_wake_pattern_cmdid = WMI_10_2_WOW_DEL_WAKE_PATTERN_CMDID, .wow_enable_disable_wake_event_cmdid = WMI_10_2_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID, .wow_enable_cmdid = WMI_10_2_WOW_ENABLE_CMDID, .wow_hostwakeup_from_sleep_cmdid = WMI_10_2_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID, .rtt_measreq_cmdid = WMI_10_2_RTT_MEASREQ_CMDID, .rtt_tsf_cmdid = WMI_10_2_RTT_TSF_CMDID, .vdev_spectral_scan_configure_cmdid = WMI_10_2_VDEV_SPECTRAL_SCAN_CONFIGURE_CMDID, .vdev_spectral_scan_enable_cmdid = WMI_10_2_VDEV_SPECTRAL_SCAN_ENABLE_CMDID, .request_stats_cmdid = WMI_10_2_REQUEST_STATS_CMDID, .set_arp_ns_offload_cmdid = WMI_CMD_UNSUPPORTED, .network_list_offload_config_cmdid = WMI_CMD_UNSUPPORTED, .gtk_offload_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_enable_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_chanswitch_cmdid = WMI_CMD_UNSUPPORTED, .chatter_set_mode_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_addba_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_delba_cmdid = WMI_CMD_UNSUPPORTED, .sta_dtim_ps_method_cmdid = WMI_CMD_UNSUPPORTED, .sta_uapsd_auto_trig_cmdid = WMI_CMD_UNSUPPORTED, .sta_keepalive_cmd = WMI_CMD_UNSUPPORTED, .echo_cmdid = WMI_10_2_ECHO_CMDID, .pdev_utf_cmdid = WMI_10_2_PDEV_UTF_CMDID, .dbglog_cfg_cmdid = WMI_10_2_DBGLOG_CFG_CMDID, .pdev_qvit_cmdid = WMI_10_2_PDEV_QVIT_CMDID, .pdev_ftm_intg_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .force_fw_hang_cmdid = WMI_CMD_UNSUPPORTED, .gpio_config_cmdid = WMI_10_2_GPIO_CONFIG_CMDID, .gpio_output_cmdid = WMI_10_2_GPIO_OUTPUT_CMDID, .pdev_get_temperature_cmdid = WMI_CMD_UNSUPPORTED, .pdev_enable_adaptive_cca_cmdid = WMI_CMD_UNSUPPORTED, .scan_update_request_cmdid = WMI_CMD_UNSUPPORTED, .vdev_standby_response_cmdid = WMI_CMD_UNSUPPORTED, .vdev_resume_response_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_add_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_evict_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_restore_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_print_all_peers_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_update_wds_entry_cmdid = WMI_CMD_UNSUPPORTED, .peer_add_proxy_sta_entry_cmdid = WMI_CMD_UNSUPPORTED, .rtt_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .oem_req_cmdid = WMI_CMD_UNSUPPORTED, .nan_cmdid = WMI_CMD_UNSUPPORTED, .vdev_ratemask_cmdid = WMI_CMD_UNSUPPORTED, .qboost_cfg_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_enable_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_set_rx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_tx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_train_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_node_config_ops_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_antenna_switch_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_ctl_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_mimogain_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_chainmsk_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_fips_cmdid = WMI_CMD_UNSUPPORTED, .tt_set_conf_cmdid = WMI_CMD_UNSUPPORTED, .fwtest_cmdid = WMI_CMD_UNSUPPORTED, .vdev_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .peer_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_cck_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_ofdm_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_reserve_ast_entry_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED, .radar_found_cmdid = WMI_CMD_UNSUPPORTED, }; static struct wmi_pdev_param_map wmi_10_4_pdev_param_map = { .tx_chain_mask = WMI_10_4_PDEV_PARAM_TX_CHAIN_MASK, .rx_chain_mask = WMI_10_4_PDEV_PARAM_RX_CHAIN_MASK, .txpower_limit2g = WMI_10_4_PDEV_PARAM_TXPOWER_LIMIT2G, .txpower_limit5g = WMI_10_4_PDEV_PARAM_TXPOWER_LIMIT5G, .txpower_scale = WMI_10_4_PDEV_PARAM_TXPOWER_SCALE, .beacon_gen_mode = WMI_10_4_PDEV_PARAM_BEACON_GEN_MODE, .beacon_tx_mode = WMI_10_4_PDEV_PARAM_BEACON_TX_MODE, .resmgr_offchan_mode = WMI_10_4_PDEV_PARAM_RESMGR_OFFCHAN_MODE, .protection_mode = WMI_10_4_PDEV_PARAM_PROTECTION_MODE, .dynamic_bw = WMI_10_4_PDEV_PARAM_DYNAMIC_BW, .non_agg_sw_retry_th = WMI_10_4_PDEV_PARAM_NON_AGG_SW_RETRY_TH, .agg_sw_retry_th = WMI_10_4_PDEV_PARAM_AGG_SW_RETRY_TH, .sta_kickout_th = WMI_10_4_PDEV_PARAM_STA_KICKOUT_TH, .ac_aggrsize_scaling = WMI_10_4_PDEV_PARAM_AC_AGGRSIZE_SCALING, .ltr_enable = WMI_10_4_PDEV_PARAM_LTR_ENABLE, .ltr_ac_latency_be = WMI_10_4_PDEV_PARAM_LTR_AC_LATENCY_BE, .ltr_ac_latency_bk = WMI_10_4_PDEV_PARAM_LTR_AC_LATENCY_BK, .ltr_ac_latency_vi = WMI_10_4_PDEV_PARAM_LTR_AC_LATENCY_VI, .ltr_ac_latency_vo = WMI_10_4_PDEV_PARAM_LTR_AC_LATENCY_VO, .ltr_ac_latency_timeout = WMI_10_4_PDEV_PARAM_LTR_AC_LATENCY_TIMEOUT, .ltr_sleep_override = WMI_10_4_PDEV_PARAM_LTR_SLEEP_OVERRIDE, .ltr_rx_override = WMI_10_4_PDEV_PARAM_LTR_RX_OVERRIDE, .ltr_tx_activity_timeout = WMI_10_4_PDEV_PARAM_LTR_TX_ACTIVITY_TIMEOUT, .l1ss_enable = WMI_10_4_PDEV_PARAM_L1SS_ENABLE, .dsleep_enable = WMI_10_4_PDEV_PARAM_DSLEEP_ENABLE, .pcielp_txbuf_flush = WMI_10_4_PDEV_PARAM_PCIELP_TXBUF_FLUSH, .pcielp_txbuf_watermark = WMI_10_4_PDEV_PARAM_PCIELP_TXBUF_WATERMARK, .pcielp_txbuf_tmo_en = WMI_10_4_PDEV_PARAM_PCIELP_TXBUF_TMO_EN, .pcielp_txbuf_tmo_value = WMI_10_4_PDEV_PARAM_PCIELP_TXBUF_TMO_VALUE, .pdev_stats_update_period = WMI_10_4_PDEV_PARAM_PDEV_STATS_UPDATE_PERIOD, .vdev_stats_update_period = WMI_10_4_PDEV_PARAM_VDEV_STATS_UPDATE_PERIOD, .peer_stats_update_period = WMI_10_4_PDEV_PARAM_PEER_STATS_UPDATE_PERIOD, .bcnflt_stats_update_period = WMI_10_4_PDEV_PARAM_BCNFLT_STATS_UPDATE_PERIOD, .pmf_qos = WMI_10_4_PDEV_PARAM_PMF_QOS, .arp_ac_override = WMI_10_4_PDEV_PARAM_ARP_AC_OVERRIDE, .dcs = WMI_10_4_PDEV_PARAM_DCS, .ani_enable = WMI_10_4_PDEV_PARAM_ANI_ENABLE, .ani_poll_period = WMI_10_4_PDEV_PARAM_ANI_POLL_PERIOD, .ani_listen_period = WMI_10_4_PDEV_PARAM_ANI_LISTEN_PERIOD, .ani_ofdm_level = WMI_10_4_PDEV_PARAM_ANI_OFDM_LEVEL, .ani_cck_level = WMI_10_4_PDEV_PARAM_ANI_CCK_LEVEL, .dyntxchain = WMI_10_4_PDEV_PARAM_DYNTXCHAIN, .proxy_sta = WMI_10_4_PDEV_PARAM_PROXY_STA, .idle_ps_config = WMI_10_4_PDEV_PARAM_IDLE_PS_CONFIG, .power_gating_sleep = WMI_10_4_PDEV_PARAM_POWER_GATING_SLEEP, .fast_channel_reset = WMI_10_4_PDEV_PARAM_FAST_CHANNEL_RESET, .burst_dur = WMI_10_4_PDEV_PARAM_BURST_DUR, .burst_enable = WMI_10_4_PDEV_PARAM_BURST_ENABLE, .cal_period = WMI_10_4_PDEV_PARAM_CAL_PERIOD, .aggr_burst = WMI_10_4_PDEV_PARAM_AGGR_BURST, .rx_decap_mode = WMI_10_4_PDEV_PARAM_RX_DECAP_MODE, .smart_antenna_default_antenna = WMI_10_4_PDEV_PARAM_SMART_ANTENNA_DEFAULT_ANTENNA, .igmpmld_override = WMI_10_4_PDEV_PARAM_IGMPMLD_OVERRIDE, .igmpmld_tid = WMI_10_4_PDEV_PARAM_IGMPMLD_TID, .antenna_gain = WMI_10_4_PDEV_PARAM_ANTENNA_GAIN, .rx_filter = WMI_10_4_PDEV_PARAM_RX_FILTER, .set_mcast_to_ucast_tid = WMI_10_4_PDEV_SET_MCAST_TO_UCAST_TID, .proxy_sta_mode = WMI_10_4_PDEV_PARAM_PROXY_STA_MODE, .set_mcast2ucast_mode = WMI_10_4_PDEV_PARAM_SET_MCAST2UCAST_MODE, .set_mcast2ucast_buffer = WMI_10_4_PDEV_PARAM_SET_MCAST2UCAST_BUFFER, .remove_mcast2ucast_buffer = WMI_10_4_PDEV_PARAM_REMOVE_MCAST2UCAST_BUFFER, .peer_sta_ps_statechg_enable = WMI_10_4_PDEV_PEER_STA_PS_STATECHG_ENABLE, .igmpmld_ac_override = WMI_10_4_PDEV_PARAM_IGMPMLD_AC_OVERRIDE, .block_interbss = WMI_10_4_PDEV_PARAM_BLOCK_INTERBSS, .set_disable_reset_cmdid = WMI_10_4_PDEV_PARAM_SET_DISABLE_RESET_CMDID, .set_msdu_ttl_cmdid = WMI_10_4_PDEV_PARAM_SET_MSDU_TTL_CMDID, .set_ppdu_duration_cmdid = WMI_10_4_PDEV_PARAM_SET_PPDU_DURATION_CMDID, .txbf_sound_period_cmdid = WMI_10_4_PDEV_PARAM_TXBF_SOUND_PERIOD_CMDID, .set_promisc_mode_cmdid = WMI_10_4_PDEV_PARAM_SET_PROMISC_MODE_CMDID, .set_burst_mode_cmdid = WMI_10_4_PDEV_PARAM_SET_BURST_MODE_CMDID, .en_stats = WMI_10_4_PDEV_PARAM_EN_STATS, .mu_group_policy = WMI_10_4_PDEV_PARAM_MU_GROUP_POLICY, .noise_detection = WMI_10_4_PDEV_PARAM_NOISE_DETECTION, .noise_threshold = WMI_10_4_PDEV_PARAM_NOISE_THRESHOLD, .dpd_enable = WMI_10_4_PDEV_PARAM_DPD_ENABLE, .set_mcast_bcast_echo = WMI_10_4_PDEV_PARAM_SET_MCAST_BCAST_ECHO, .atf_strict_sch = WMI_10_4_PDEV_PARAM_ATF_STRICT_SCH, .atf_sched_duration = WMI_10_4_PDEV_PARAM_ATF_SCHED_DURATION, .ant_plzn = WMI_10_4_PDEV_PARAM_ANT_PLZN, .mgmt_retry_limit = WMI_10_4_PDEV_PARAM_MGMT_RETRY_LIMIT, .sensitivity_level = WMI_10_4_PDEV_PARAM_SENSITIVITY_LEVEL, .signed_txpower_2g = WMI_10_4_PDEV_PARAM_SIGNED_TXPOWER_2G, .signed_txpower_5g = WMI_10_4_PDEV_PARAM_SIGNED_TXPOWER_5G, .enable_per_tid_amsdu = WMI_10_4_PDEV_PARAM_ENABLE_PER_TID_AMSDU, .enable_per_tid_ampdu = WMI_10_4_PDEV_PARAM_ENABLE_PER_TID_AMPDU, .cca_threshold = WMI_10_4_PDEV_PARAM_CCA_THRESHOLD, .rts_fixed_rate = WMI_10_4_PDEV_PARAM_RTS_FIXED_RATE, .pdev_reset = WMI_10_4_PDEV_PARAM_PDEV_RESET, .wapi_mbssid_offset = WMI_10_4_PDEV_PARAM_WAPI_MBSSID_OFFSET, .arp_srcaddr = WMI_10_4_PDEV_PARAM_ARP_SRCADDR, .arp_dstaddr = WMI_10_4_PDEV_PARAM_ARP_DSTADDR, .enable_btcoex = WMI_10_4_PDEV_PARAM_ENABLE_BTCOEX, }; static const u8 wmi_key_cipher_suites[] = { [WMI_CIPHER_NONE] = WMI_CIPHER_NONE, [WMI_CIPHER_WEP] = WMI_CIPHER_WEP, [WMI_CIPHER_TKIP] = WMI_CIPHER_TKIP, [WMI_CIPHER_AES_OCB] = WMI_CIPHER_AES_OCB, [WMI_CIPHER_AES_CCM] = WMI_CIPHER_AES_CCM, [WMI_CIPHER_WAPI] = WMI_CIPHER_WAPI, [WMI_CIPHER_CKIP] = WMI_CIPHER_CKIP, [WMI_CIPHER_AES_CMAC] = WMI_CIPHER_AES_CMAC, [WMI_CIPHER_AES_GCM] = WMI_CIPHER_AES_GCM, }; static const u8 wmi_tlv_key_cipher_suites[] = { [WMI_CIPHER_NONE] = WMI_TLV_CIPHER_NONE, [WMI_CIPHER_WEP] = WMI_TLV_CIPHER_WEP, [WMI_CIPHER_TKIP] = WMI_TLV_CIPHER_TKIP, [WMI_CIPHER_AES_OCB] = WMI_TLV_CIPHER_AES_OCB, [WMI_CIPHER_AES_CCM] = WMI_TLV_CIPHER_AES_CCM, [WMI_CIPHER_WAPI] = WMI_TLV_CIPHER_WAPI, [WMI_CIPHER_CKIP] = WMI_TLV_CIPHER_CKIP, [WMI_CIPHER_AES_CMAC] = WMI_TLV_CIPHER_AES_CMAC, [WMI_CIPHER_AES_GCM] = WMI_TLV_CIPHER_AES_GCM, }; static const struct wmi_peer_flags_map wmi_peer_flags_map = { .auth = WMI_PEER_AUTH, .qos = WMI_PEER_QOS, .need_ptk_4_way = WMI_PEER_NEED_PTK_4_WAY, .need_gtk_2_way = WMI_PEER_NEED_GTK_2_WAY, .apsd = WMI_PEER_APSD, .ht = WMI_PEER_HT, .bw40 = WMI_PEER_40MHZ, .stbc = WMI_PEER_STBC, .ldbc = WMI_PEER_LDPC, .dyn_mimops = WMI_PEER_DYN_MIMOPS, .static_mimops = WMI_PEER_STATIC_MIMOPS, .spatial_mux = WMI_PEER_SPATIAL_MUX, .vht = WMI_PEER_VHT, .bw80 = WMI_PEER_80MHZ, .vht_2g = WMI_PEER_VHT_2G, .pmf = WMI_PEER_PMF, .bw160 = WMI_PEER_160MHZ, }; static const struct wmi_peer_flags_map wmi_10x_peer_flags_map = { .auth = WMI_10X_PEER_AUTH, .qos = WMI_10X_PEER_QOS, .need_ptk_4_way = WMI_10X_PEER_NEED_PTK_4_WAY, .need_gtk_2_way = WMI_10X_PEER_NEED_GTK_2_WAY, .apsd = WMI_10X_PEER_APSD, .ht = WMI_10X_PEER_HT, .bw40 = WMI_10X_PEER_40MHZ, .stbc = WMI_10X_PEER_STBC, .ldbc = WMI_10X_PEER_LDPC, .dyn_mimops = WMI_10X_PEER_DYN_MIMOPS, .static_mimops = WMI_10X_PEER_STATIC_MIMOPS, .spatial_mux = WMI_10X_PEER_SPATIAL_MUX, .vht = WMI_10X_PEER_VHT, .bw80 = WMI_10X_PEER_80MHZ, .bw160 = WMI_10X_PEER_160MHZ, }; static const struct wmi_peer_flags_map wmi_10_2_peer_flags_map = { .auth = WMI_10_2_PEER_AUTH, .qos = WMI_10_2_PEER_QOS, .need_ptk_4_way = WMI_10_2_PEER_NEED_PTK_4_WAY, .need_gtk_2_way = WMI_10_2_PEER_NEED_GTK_2_WAY, .apsd = WMI_10_2_PEER_APSD, .ht = WMI_10_2_PEER_HT, .bw40 = WMI_10_2_PEER_40MHZ, .stbc = WMI_10_2_PEER_STBC, .ldbc = WMI_10_2_PEER_LDPC, .dyn_mimops = WMI_10_2_PEER_DYN_MIMOPS, .static_mimops = WMI_10_2_PEER_STATIC_MIMOPS, .spatial_mux = WMI_10_2_PEER_SPATIAL_MUX, .vht = WMI_10_2_PEER_VHT, .bw80 = WMI_10_2_PEER_80MHZ, .vht_2g = WMI_10_2_PEER_VHT_2G, .pmf = WMI_10_2_PEER_PMF, .bw160 = WMI_10_2_PEER_160MHZ, }; void ath10k_wmi_put_wmi_channel(struct ath10k *ar, struct wmi_channel *ch, const struct wmi_channel_arg *arg) { u32 flags = 0; struct ieee80211_channel *chan = NULL; memset(ch, 0, sizeof(*ch)); if (arg->passive) flags |= WMI_CHAN_FLAG_PASSIVE; if (arg->allow_ibss) flags |= WMI_CHAN_FLAG_ADHOC_ALLOWED; if (arg->allow_ht) flags |= WMI_CHAN_FLAG_ALLOW_HT; if (arg->allow_vht) flags |= WMI_CHAN_FLAG_ALLOW_VHT; if (arg->ht40plus) flags |= WMI_CHAN_FLAG_HT40_PLUS; if (arg->chan_radar) flags |= WMI_CHAN_FLAG_DFS; ch->band_center_freq2 = 0; ch->mhz = __cpu_to_le32(arg->freq); ch->band_center_freq1 = __cpu_to_le32(arg->band_center_freq1); if (arg->mode == MODE_11AC_VHT80_80) { ch->band_center_freq2 = __cpu_to_le32(arg->band_center_freq2); chan = ieee80211_get_channel(ar->hw->wiphy, arg->band_center_freq2 - 10); } if (arg->mode == MODE_11AC_VHT160) { u32 band_center_freq1; u32 band_center_freq2; if (arg->freq > arg->band_center_freq1) { band_center_freq1 = arg->band_center_freq1 + 40; band_center_freq2 = arg->band_center_freq1 - 40; } else { band_center_freq1 = arg->band_center_freq1 - 40; band_center_freq2 = arg->band_center_freq1 + 40; } ch->band_center_freq1 = __cpu_to_le32(band_center_freq1); /* Minus 10 to get a defined 5G channel frequency*/ chan = ieee80211_get_channel(ar->hw->wiphy, band_center_freq2 - 10); /* The center frequency of the entire VHT160 */ ch->band_center_freq2 = __cpu_to_le32(arg->band_center_freq1); } if (chan && chan->flags & IEEE80211_CHAN_RADAR) flags |= WMI_CHAN_FLAG_DFS_CFREQ2; ch->min_power = arg->min_power; ch->max_power = arg->max_power; ch->reg_power = arg->max_reg_power; ch->antenna_max = arg->max_antenna_gain; ch->max_tx_power = arg->max_power; /* mode & flags share storage */ ch->mode = arg->mode; ch->flags |= __cpu_to_le32(flags); } int ath10k_wmi_wait_for_service_ready(struct ath10k *ar) { unsigned long time_left, i; time_left = wait_for_completion_timeout(&ar->wmi.service_ready, WMI_SERVICE_READY_TIMEOUT_HZ); if (!time_left) { /* Sometimes the PCI HIF doesn't receive interrupt * for the service ready message even if the buffer * was completed. PCIe sniffer shows that it's * because the corresponding CE ring doesn't fires * it. Workaround here by polling CE rings once. */ ath10k_warn(ar, "failed to receive service ready completion, polling..\n"); for (i = 0; i < CE_COUNT; i++) ath10k_hif_send_complete_check(ar, i, 1); time_left = wait_for_completion_timeout(&ar->wmi.service_ready, WMI_SERVICE_READY_TIMEOUT_HZ); if (!time_left) { ath10k_warn(ar, "polling timed out\n"); return -ETIMEDOUT; } ath10k_warn(ar, "service ready completion received, continuing normally\n"); } return 0; } int ath10k_wmi_wait_for_unified_ready(struct ath10k *ar) { unsigned long time_left; time_left = wait_for_completion_timeout(&ar->wmi.unified_ready, WMI_UNIFIED_READY_TIMEOUT_HZ); if (!time_left) return -ETIMEDOUT; return 0; } struct sk_buff *ath10k_wmi_alloc_skb(struct ath10k *ar, u32 len) { struct sk_buff *skb; u32 round_len = roundup(len, 4); skb = ath10k_htc_alloc_skb(ar, WMI_SKB_HEADROOM + round_len); if (!skb) return NULL; skb_reserve(skb, WMI_SKB_HEADROOM); if (!IS_ALIGNED((unsigned long)skb->data, 4)) ath10k_warn(ar, "Unaligned WMI skb\n"); skb_put(skb, round_len); memset(skb->data, 0, round_len); return skb; } static void ath10k_wmi_htc_tx_complete(struct ath10k *ar, struct sk_buff *skb) { dev_kfree_skb(skb); } int ath10k_wmi_cmd_send_nowait(struct ath10k *ar, struct sk_buff *skb, u32 cmd_id) { struct ath10k_skb_cb *skb_cb = ATH10K_SKB_CB(skb); struct wmi_cmd_hdr *cmd_hdr; int ret; u32 cmd = 0; if (skb_push(skb, sizeof(struct wmi_cmd_hdr)) == NULL) return -ENOMEM; cmd |= SM(cmd_id, WMI_CMD_HDR_CMD_ID); cmd_hdr = (struct wmi_cmd_hdr *)skb->data; cmd_hdr->cmd_id = __cpu_to_le32(cmd); memset(skb_cb, 0, sizeof(*skb_cb)); trace_ath10k_wmi_cmd(ar, cmd_id, skb->data, skb->len); ret = ath10k_htc_send(&ar->htc, ar->wmi.eid, skb); if (ret) goto err_pull; return 0; err_pull: skb_pull(skb, sizeof(struct wmi_cmd_hdr)); return ret; } static void ath10k_wmi_tx_beacon_nowait(struct ath10k_vif *arvif) { struct ath10k *ar = arvif->ar; struct ath10k_skb_cb *cb; struct sk_buff *bcn; bool dtim_zero; bool deliver_cab; int ret; spin_lock_bh(&ar->data_lock); bcn = arvif->beacon; if (!bcn) goto unlock; cb = ATH10K_SKB_CB(bcn); switch (arvif->beacon_state) { case ATH10K_BEACON_SENDING: case ATH10K_BEACON_SENT: break; case ATH10K_BEACON_SCHEDULED: arvif->beacon_state = ATH10K_BEACON_SENDING; spin_unlock_bh(&ar->data_lock); dtim_zero = !!(cb->flags & ATH10K_SKB_F_DTIM_ZERO); deliver_cab = !!(cb->flags & ATH10K_SKB_F_DELIVER_CAB); ret = ath10k_wmi_beacon_send_ref_nowait(arvif->ar, arvif->vdev_id, bcn->data, bcn->len, cb->paddr, dtim_zero, deliver_cab); spin_lock_bh(&ar->data_lock); if (ret == 0) arvif->beacon_state = ATH10K_BEACON_SENT; else arvif->beacon_state = ATH10K_BEACON_SCHEDULED; } unlock: spin_unlock_bh(&ar->data_lock); } static void ath10k_wmi_tx_beacons_iter(void *data, u8 *mac, struct ieee80211_vif *vif) { struct ath10k_vif *arvif = (void *)vif->drv_priv; ath10k_wmi_tx_beacon_nowait(arvif); } static void ath10k_wmi_tx_beacons_nowait(struct ath10k *ar) { ieee80211_iterate_active_interfaces_atomic(ar->hw, ATH10K_ITER_NORMAL_FLAGS, ath10k_wmi_tx_beacons_iter, NULL); } static void ath10k_wmi_op_ep_tx_credits(struct ath10k *ar) { /* try to send pending beacons first. they take priority */ ath10k_wmi_tx_beacons_nowait(ar); wake_up(&ar->wmi.tx_credits_wq); } int ath10k_wmi_cmd_send(struct ath10k *ar, struct sk_buff *skb, u32 cmd_id) { int ret = -EOPNOTSUPP; might_sleep(); if (cmd_id == WMI_CMD_UNSUPPORTED) { ath10k_warn(ar, "wmi command %d is not supported by firmware\n", cmd_id); dev_kfree_skb_any(skb); return ret; } wait_event_timeout(ar->wmi.tx_credits_wq, ({ if (ar->state == ATH10K_STATE_WEDGED) { ret = -ESHUTDOWN; ath10k_dbg(ar, ATH10K_DBG_WMI, "drop wmi command %d, hardware is wedged\n", cmd_id); } /* try to send pending beacons first. they take priority */ ath10k_wmi_tx_beacons_nowait(ar); ret = ath10k_wmi_cmd_send_nowait(ar, skb, cmd_id); if (ret && test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags)) ret = -ESHUTDOWN; (ret != -EAGAIN); }), 3 * HZ); if (ret) dev_kfree_skb_any(skb); if (ret == -EAGAIN) { ath10k_warn(ar, "wmi command %d timeout, restarting hardware\n", cmd_id); ath10k_core_start_recovery(ar); } return ret; } static struct sk_buff * ath10k_wmi_op_gen_mgmt_tx(struct ath10k *ar, struct sk_buff *msdu) { struct ath10k_skb_cb *cb = ATH10K_SKB_CB(msdu); struct ath10k_vif *arvif; struct wmi_mgmt_tx_cmd *cmd; struct ieee80211_hdr *hdr; struct sk_buff *skb; int len; u32 vdev_id; u32 buf_len = msdu->len; u16 fc; const u8 *peer_addr; hdr = (struct ieee80211_hdr *)msdu->data; fc = le16_to_cpu(hdr->frame_control); if (cb->vif) { arvif = (void *)cb->vif->drv_priv; vdev_id = arvif->vdev_id; } else { vdev_id = 0; } if (WARN_ON_ONCE(!ieee80211_is_mgmt(hdr->frame_control))) return ERR_PTR(-EINVAL); len = sizeof(cmd->hdr) + msdu->len; if ((ieee80211_is_action(hdr->frame_control) || ieee80211_is_deauth(hdr->frame_control) || ieee80211_is_disassoc(hdr->frame_control)) && ieee80211_has_protected(hdr->frame_control)) { peer_addr = hdr->addr1; if (is_multicast_ether_addr(peer_addr)) { len += sizeof(struct ieee80211_mmie_16); buf_len += sizeof(struct ieee80211_mmie_16); } else { if (cb->ucast_cipher == WLAN_CIPHER_SUITE_GCMP || cb->ucast_cipher == WLAN_CIPHER_SUITE_GCMP_256) { len += IEEE80211_GCMP_MIC_LEN; buf_len += IEEE80211_GCMP_MIC_LEN; } else { len += IEEE80211_CCMP_MIC_LEN; buf_len += IEEE80211_CCMP_MIC_LEN; } } } len = round_up(len, 4); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_mgmt_tx_cmd *)skb->data; cmd->hdr.vdev_id = __cpu_to_le32(vdev_id); cmd->hdr.tx_rate = 0; cmd->hdr.tx_power = 0; cmd->hdr.buf_len = __cpu_to_le32(buf_len); ether_addr_copy(cmd->hdr.peer_macaddr.addr, ieee80211_get_DA(hdr)); memcpy(cmd->buf, msdu->data, msdu->len); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi mgmt tx skb %p len %d ftype %02x stype %02x\n", msdu, skb->len, fc & IEEE80211_FCTL_FTYPE, fc & IEEE80211_FCTL_STYPE); trace_ath10k_tx_hdr(ar, skb->data, skb->len); trace_ath10k_tx_payload(ar, skb->data, skb->len); return skb; } static void ath10k_wmi_event_scan_started(struct ath10k *ar) { lockdep_assert_held(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: ath10k_warn(ar, "received scan started event in an invalid scan state: %s (%d)\n", ath10k_scan_state_str(ar->scan.state), ar->scan.state); break; case ATH10K_SCAN_STARTING: ar->scan.state = ATH10K_SCAN_RUNNING; if (ar->scan.is_roc) ieee80211_ready_on_channel(ar->hw); complete(&ar->scan.started); break; } } static void ath10k_wmi_event_scan_start_failed(struct ath10k *ar) { lockdep_assert_held(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: ath10k_warn(ar, "received scan start failed event in an invalid scan state: %s (%d)\n", ath10k_scan_state_str(ar->scan.state), ar->scan.state); break; case ATH10K_SCAN_STARTING: complete(&ar->scan.started); __ath10k_scan_finish(ar); break; } } static void ath10k_wmi_event_scan_completed(struct ath10k *ar) { lockdep_assert_held(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_STARTING: /* One suspected reason scan can be completed while starting is * if firmware fails to deliver all scan events to the host, * e.g. when transport pipe is full. This has been observed * with spectral scan phyerr events starving wmi transport * pipe. In such case the "scan completed" event should be (and * is) ignored by the host as it may be just firmware's scan * state machine recovering. */ ath10k_warn(ar, "received scan completed event in an invalid scan state: %s (%d)\n", ath10k_scan_state_str(ar->scan.state), ar->scan.state); break; case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: __ath10k_scan_finish(ar); break; } } static void ath10k_wmi_event_scan_bss_chan(struct ath10k *ar) { lockdep_assert_held(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_STARTING: ath10k_warn(ar, "received scan bss chan event in an invalid scan state: %s (%d)\n", ath10k_scan_state_str(ar->scan.state), ar->scan.state); break; case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: ar->scan_channel = NULL; break; } } static void ath10k_wmi_event_scan_foreign_chan(struct ath10k *ar, u32 freq) { lockdep_assert_held(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_STARTING: ath10k_warn(ar, "received scan foreign chan event in an invalid scan state: %s (%d)\n", ath10k_scan_state_str(ar->scan.state), ar->scan.state); break; case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: ar->scan_channel = ieee80211_get_channel(ar->hw->wiphy, freq); if (ar->scan.is_roc && ar->scan.roc_freq == freq) complete(&ar->scan.on_channel); break; } } static const char * ath10k_wmi_event_scan_type_str(enum wmi_scan_event_type type, enum wmi_scan_completion_reason reason) { switch (type) { case WMI_SCAN_EVENT_STARTED: return "started"; case WMI_SCAN_EVENT_COMPLETED: switch (reason) { case WMI_SCAN_REASON_COMPLETED: return "completed"; case WMI_SCAN_REASON_CANCELLED: return "completed [cancelled]"; case WMI_SCAN_REASON_PREEMPTED: return "completed [preempted]"; case WMI_SCAN_REASON_TIMEDOUT: return "completed [timedout]"; case WMI_SCAN_REASON_INTERNAL_FAILURE: return "completed [internal err]"; case WMI_SCAN_REASON_MAX: break; } return "completed [unknown]"; case WMI_SCAN_EVENT_BSS_CHANNEL: return "bss channel"; case WMI_SCAN_EVENT_FOREIGN_CHANNEL: return "foreign channel"; case WMI_SCAN_EVENT_DEQUEUED: return "dequeued"; case WMI_SCAN_EVENT_PREEMPTED: return "preempted"; case WMI_SCAN_EVENT_START_FAILED: return "start failed"; case WMI_SCAN_EVENT_RESTARTED: return "restarted"; case WMI_SCAN_EVENT_FOREIGN_CHANNEL_EXIT: return "foreign channel exit"; default: return "unknown"; } } static int ath10k_wmi_op_pull_scan_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_scan_ev_arg *arg) { struct wmi_scan_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->event_type = ev->event_type; arg->reason = ev->reason; arg->channel_freq = ev->channel_freq; arg->scan_req_id = ev->scan_req_id; arg->scan_id = ev->scan_id; arg->vdev_id = ev->vdev_id; return 0; } int ath10k_wmi_event_scan(struct ath10k *ar, struct sk_buff *skb) { struct wmi_scan_ev_arg arg = {}; enum wmi_scan_event_type event_type; enum wmi_scan_completion_reason reason; u32 freq; u32 req_id; u32 scan_id; u32 vdev_id; int ret; ret = ath10k_wmi_pull_scan(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse scan event: %d\n", ret); return ret; } event_type = __le32_to_cpu(arg.event_type); reason = __le32_to_cpu(arg.reason); freq = __le32_to_cpu(arg.channel_freq); req_id = __le32_to_cpu(arg.scan_req_id); scan_id = __le32_to_cpu(arg.scan_id); vdev_id = __le32_to_cpu(arg.vdev_id); spin_lock_bh(&ar->data_lock); ath10k_dbg(ar, ATH10K_DBG_WMI, "scan event %s type %d reason %d freq %d req_id %d scan_id %d vdev_id %d state %s (%d)\n", ath10k_wmi_event_scan_type_str(event_type, reason), event_type, reason, freq, req_id, scan_id, vdev_id, ath10k_scan_state_str(ar->scan.state), ar->scan.state); switch (event_type) { case WMI_SCAN_EVENT_STARTED: ath10k_wmi_event_scan_started(ar); break; case WMI_SCAN_EVENT_COMPLETED: ath10k_wmi_event_scan_completed(ar); break; case WMI_SCAN_EVENT_BSS_CHANNEL: ath10k_wmi_event_scan_bss_chan(ar); break; case WMI_SCAN_EVENT_FOREIGN_CHANNEL: ath10k_wmi_event_scan_foreign_chan(ar, freq); break; case WMI_SCAN_EVENT_START_FAILED: ath10k_warn(ar, "received scan start failure event\n"); ath10k_wmi_event_scan_start_failed(ar); break; case WMI_SCAN_EVENT_DEQUEUED: case WMI_SCAN_EVENT_PREEMPTED: case WMI_SCAN_EVENT_RESTARTED: case WMI_SCAN_EVENT_FOREIGN_CHANNEL_EXIT: default: break; } spin_unlock_bh(&ar->data_lock); return 0; } /* If keys are configured, HW decrypts all frames * with protected bit set. Mark such frames as decrypted. */ static void ath10k_wmi_handle_wep_reauth(struct ath10k *ar, struct sk_buff *skb, struct ieee80211_rx_status *status) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; unsigned int hdrlen; bool peer_key; u8 *addr, keyidx; if (!ieee80211_is_auth(hdr->frame_control) || !ieee80211_has_protected(hdr->frame_control)) return; hdrlen = ieee80211_hdrlen(hdr->frame_control); if (skb->len < (hdrlen + IEEE80211_WEP_IV_LEN)) return; keyidx = skb->data[hdrlen + (IEEE80211_WEP_IV_LEN - 1)] >> WEP_KEYID_SHIFT; addr = ieee80211_get_SA(hdr); spin_lock_bh(&ar->data_lock); peer_key = ath10k_mac_is_peer_wep_key_set(ar, addr, keyidx); spin_unlock_bh(&ar->data_lock); if (peer_key) { ath10k_dbg(ar, ATH10K_DBG_MAC, "mac wep key present for peer %pM\n", addr); status->flag |= RX_FLAG_DECRYPTED; } } static int ath10k_wmi_op_pull_mgmt_rx_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_mgmt_rx_ev_arg *arg) { struct wmi_mgmt_rx_event_v1 *ev_v1; struct wmi_mgmt_rx_event_v2 *ev_v2; struct wmi_mgmt_rx_hdr_v1 *ev_hdr; struct wmi_mgmt_rx_ext_info *ext_info; size_t pull_len; u32 msdu_len; u32 len; if (test_bit(ATH10K_FW_FEATURE_EXT_WMI_MGMT_RX, ar->running_fw->fw_file.fw_features)) { ev_v2 = (struct wmi_mgmt_rx_event_v2 *)skb->data; ev_hdr = &ev_v2->hdr.v1; pull_len = sizeof(*ev_v2); } else { ev_v1 = (struct wmi_mgmt_rx_event_v1 *)skb->data; ev_hdr = &ev_v1->hdr; pull_len = sizeof(*ev_v1); } if (skb->len < pull_len) return -EPROTO; skb_pull(skb, pull_len); arg->channel = ev_hdr->channel; arg->buf_len = ev_hdr->buf_len; arg->status = ev_hdr->status; arg->snr = ev_hdr->snr; arg->phy_mode = ev_hdr->phy_mode; arg->rate = ev_hdr->rate; msdu_len = __le32_to_cpu(arg->buf_len); if (skb->len < msdu_len) return -EPROTO; if (le32_to_cpu(arg->status) & WMI_RX_STATUS_EXT_INFO) { len = ALIGN(le32_to_cpu(arg->buf_len), 4); ext_info = (struct wmi_mgmt_rx_ext_info *)(skb->data + len); memcpy(&arg->ext_info, ext_info, sizeof(struct wmi_mgmt_rx_ext_info)); } /* the WMI buffer might've ended up being padded to 4 bytes due to HTC * trailer with credit update. Trim the excess garbage. */ skb_trim(skb, msdu_len); return 0; } static int ath10k_wmi_10_4_op_pull_mgmt_rx_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_mgmt_rx_ev_arg *arg) { struct wmi_10_4_mgmt_rx_event *ev; struct wmi_10_4_mgmt_rx_hdr *ev_hdr; size_t pull_len; u32 msdu_len; struct wmi_mgmt_rx_ext_info *ext_info; u32 len; ev = (struct wmi_10_4_mgmt_rx_event *)skb->data; ev_hdr = &ev->hdr; pull_len = sizeof(*ev); if (skb->len < pull_len) return -EPROTO; skb_pull(skb, pull_len); arg->channel = ev_hdr->channel; arg->buf_len = ev_hdr->buf_len; arg->status = ev_hdr->status; arg->snr = ev_hdr->snr; arg->phy_mode = ev_hdr->phy_mode; arg->rate = ev_hdr->rate; msdu_len = __le32_to_cpu(arg->buf_len); if (skb->len < msdu_len) return -EPROTO; if (le32_to_cpu(arg->status) & WMI_RX_STATUS_EXT_INFO) { len = ALIGN(le32_to_cpu(arg->buf_len), 4); ext_info = (struct wmi_mgmt_rx_ext_info *)(skb->data + len); memcpy(&arg->ext_info, ext_info, sizeof(struct wmi_mgmt_rx_ext_info)); } /* Make sure bytes added for padding are removed. */ skb_trim(skb, msdu_len); return 0; } static bool ath10k_wmi_rx_is_decrypted(struct ath10k *ar, struct ieee80211_hdr *hdr) { if (!ieee80211_has_protected(hdr->frame_control)) return false; /* FW delivers WEP Shared Auth frame with Protected Bit set and * encrypted payload. However in case of PMF it delivers decrypted * frames with Protected Bit set. */ if (ieee80211_is_auth(hdr->frame_control)) return false; /* qca99x0 based FW delivers broadcast or multicast management frames * (ex: group privacy action frames in mesh) as encrypted payload. */ if (is_multicast_ether_addr(ieee80211_get_DA(hdr)) && ar->hw_params.sw_decrypt_mcast_mgmt) return false; return true; } static int wmi_process_mgmt_tx_comp(struct ath10k *ar, struct mgmt_tx_compl_params *param) { struct ath10k_mgmt_tx_pkt_addr *pkt_addr; struct ath10k_wmi *wmi = &ar->wmi; struct ieee80211_tx_info *info; struct sk_buff *msdu; int ret; spin_lock_bh(&ar->data_lock); pkt_addr = idr_find(&wmi->mgmt_pending_tx, param->desc_id); if (!pkt_addr) { ath10k_warn(ar, "received mgmt tx completion for invalid msdu_id: %d\n", param->desc_id); ret = -ENOENT; goto out; } msdu = pkt_addr->vaddr; dma_unmap_single(ar->dev, pkt_addr->paddr, msdu->len, DMA_TO_DEVICE); info = IEEE80211_SKB_CB(msdu); kfree(pkt_addr); if (param->status) { info->flags &= ~IEEE80211_TX_STAT_ACK; } else { info->flags |= IEEE80211_TX_STAT_ACK; info->status.ack_signal = ATH10K_DEFAULT_NOISE_FLOOR + param->ack_rssi; info->status.flags |= IEEE80211_TX_STATUS_ACK_SIGNAL_VALID; } ieee80211_tx_status_irqsafe(ar->hw, msdu); ret = 0; out: idr_remove(&wmi->mgmt_pending_tx, param->desc_id); spin_unlock_bh(&ar->data_lock); return ret; } int ath10k_wmi_event_mgmt_tx_compl(struct ath10k *ar, struct sk_buff *skb) { struct wmi_tlv_mgmt_tx_compl_ev_arg arg; struct mgmt_tx_compl_params param; int ret; ret = ath10k_wmi_pull_mgmt_tx_compl(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse mgmt comp event: %d\n", ret); return ret; } memset(¶m, 0, sizeof(struct mgmt_tx_compl_params)); param.desc_id = __le32_to_cpu(arg.desc_id); param.status = __le32_to_cpu(arg.status); if (test_bit(WMI_SERVICE_TX_DATA_ACK_RSSI, ar->wmi.svc_map)) param.ack_rssi = __le32_to_cpu(arg.ack_rssi); wmi_process_mgmt_tx_comp(ar, ¶m); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi tlv evnt mgmt tx completion\n"); return 0; } int ath10k_wmi_event_mgmt_tx_bundle_compl(struct ath10k *ar, struct sk_buff *skb) { struct wmi_tlv_mgmt_tx_bundle_compl_ev_arg arg; struct mgmt_tx_compl_params param; u32 num_reports; int i, ret; ret = ath10k_wmi_pull_mgmt_tx_bundle_compl(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse bundle mgmt compl event: %d\n", ret); return ret; } num_reports = __le32_to_cpu(arg.num_reports); for (i = 0; i < num_reports; i++) { memset(¶m, 0, sizeof(struct mgmt_tx_compl_params)); param.desc_id = __le32_to_cpu(arg.desc_ids[i]); param.status = __le32_to_cpu(arg.desc_ids[i]); if (test_bit(WMI_SERVICE_TX_DATA_ACK_RSSI, ar->wmi.svc_map)) param.ack_rssi = __le32_to_cpu(arg.ack_rssi[i]); wmi_process_mgmt_tx_comp(ar, ¶m); } ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi tlv event bundle mgmt tx completion\n"); return 0; } int ath10k_wmi_event_mgmt_rx(struct ath10k *ar, struct sk_buff *skb) { struct wmi_mgmt_rx_ev_arg arg = {}; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); struct ieee80211_hdr *hdr; struct ieee80211_supported_band *sband; u32 rx_status; u32 channel; u32 phy_mode; u32 snr, rssi; u32 rate; u16 fc; int ret, i; ret = ath10k_wmi_pull_mgmt_rx(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse mgmt rx event: %d\n", ret); dev_kfree_skb(skb); return ret; } channel = __le32_to_cpu(arg.channel); rx_status = __le32_to_cpu(arg.status); snr = __le32_to_cpu(arg.snr); phy_mode = __le32_to_cpu(arg.phy_mode); rate = __le32_to_cpu(arg.rate); memset(status, 0, sizeof(*status)); ath10k_dbg(ar, ATH10K_DBG_MGMT, "event mgmt rx status %08x\n", rx_status); if ((test_bit(ATH10K_CAC_RUNNING, &ar->dev_flags)) || (rx_status & (WMI_RX_STATUS_ERR_DECRYPT | WMI_RX_STATUS_ERR_KEY_CACHE_MISS | WMI_RX_STATUS_ERR_CRC))) { dev_kfree_skb(skb); return 0; } if (rx_status & WMI_RX_STATUS_ERR_MIC) status->flag |= RX_FLAG_MMIC_ERROR; if (rx_status & WMI_RX_STATUS_EXT_INFO) { status->mactime = __le64_to_cpu(arg.ext_info.rx_mac_timestamp); status->flag |= RX_FLAG_MACTIME_END; } /* Hardware can Rx CCK rates on 5GHz. In that case phy_mode is set to * MODE_11B. This means phy_mode is not a reliable source for the band * of mgmt rx. */ if (channel >= 1 && channel <= 14) { status->band = NL80211_BAND_2GHZ; } else if (channel >= 36 && channel <= ATH10K_MAX_5G_CHAN) { status->band = NL80211_BAND_5GHZ; } else { /* Shouldn't happen unless list of advertised channels to * mac80211 has been changed. */ WARN_ON_ONCE(1); dev_kfree_skb(skb); return 0; } if (phy_mode == MODE_11B && status->band == NL80211_BAND_5GHZ) ath10k_dbg(ar, ATH10K_DBG_MGMT, "wmi mgmt rx 11b (CCK) on 5GHz\n"); sband = &ar->mac.sbands[status->band]; status->freq = ieee80211_channel_to_frequency(channel, status->band); status->signal = snr + ATH10K_DEFAULT_NOISE_FLOOR; BUILD_BUG_ON(ARRAY_SIZE(status->chain_signal) != ARRAY_SIZE(arg.rssi)); for (i = 0; i < ARRAY_SIZE(status->chain_signal); i++) { status->chains &= ~BIT(i); rssi = __le32_to_cpu(arg.rssi[i]); ath10k_dbg(ar, ATH10K_DBG_MGMT, "mgmt rssi[%d]:%d\n", i, arg.rssi[i]); if (rssi != ATH10K_INVALID_RSSI && rssi != 0) { status->chain_signal[i] = ATH10K_DEFAULT_NOISE_FLOOR + rssi; status->chains |= BIT(i); } } status->rate_idx = ath10k_mac_bitrate_to_idx(sband, rate / 100); hdr = (struct ieee80211_hdr *)skb->data; fc = le16_to_cpu(hdr->frame_control); /* Firmware is guaranteed to report all essential management frames via * WMI while it can deliver some extra via HTT. Since there can be * duplicates split the reporting wrt monitor/sniffing. */ status->flag |= RX_FLAG_SKIP_MONITOR; ath10k_wmi_handle_wep_reauth(ar, skb, status); if (ath10k_wmi_rx_is_decrypted(ar, hdr)) { status->flag |= RX_FLAG_DECRYPTED; if (!ieee80211_is_action(hdr->frame_control) && !ieee80211_is_deauth(hdr->frame_control) && !ieee80211_is_disassoc(hdr->frame_control)) { status->flag |= RX_FLAG_IV_STRIPPED | RX_FLAG_MMIC_STRIPPED; hdr->frame_control = __cpu_to_le16(fc & ~IEEE80211_FCTL_PROTECTED); } } if (ieee80211_is_beacon(hdr->frame_control)) ath10k_mac_handle_beacon(ar, skb); if (ieee80211_is_beacon(hdr->frame_control) || ieee80211_is_probe_resp(hdr->frame_control)) status->boottime_ns = ktime_get_boottime_ns(); ath10k_dbg(ar, ATH10K_DBG_MGMT, "event mgmt rx skb %p len %d ftype %02x stype %02x\n", skb, skb->len, fc & IEEE80211_FCTL_FTYPE, fc & IEEE80211_FCTL_STYPE); ath10k_dbg(ar, ATH10K_DBG_MGMT, "event mgmt rx freq %d band %d snr %d, rate_idx %d\n", status->freq, status->band, status->signal, status->rate_idx); ieee80211_rx_ni(ar->hw, skb); return 0; } static int freq_to_idx(struct ath10k *ar, int freq) { struct ieee80211_supported_band *sband; int band, ch, idx = 0; for (band = NL80211_BAND_2GHZ; band < NUM_NL80211_BANDS; band++) { sband = ar->hw->wiphy->bands[band]; if (!sband) continue; for (ch = 0; ch < sband->n_channels; ch++, idx++) if (sband->channels[ch].center_freq == freq) goto exit; } exit: return idx; } static int ath10k_wmi_op_pull_ch_info_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_ch_info_ev_arg *arg) { struct wmi_chan_info_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->err_code = ev->err_code; arg->freq = ev->freq; arg->cmd_flags = ev->cmd_flags; arg->noise_floor = ev->noise_floor; arg->rx_clear_count = ev->rx_clear_count; arg->cycle_count = ev->cycle_count; return 0; } static int ath10k_wmi_10_4_op_pull_ch_info_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_ch_info_ev_arg *arg) { struct wmi_10_4_chan_info_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->err_code = ev->err_code; arg->freq = ev->freq; arg->cmd_flags = ev->cmd_flags; arg->noise_floor = ev->noise_floor; arg->rx_clear_count = ev->rx_clear_count; arg->cycle_count = ev->cycle_count; arg->chan_tx_pwr_range = ev->chan_tx_pwr_range; arg->chan_tx_pwr_tp = ev->chan_tx_pwr_tp; arg->rx_frame_count = ev->rx_frame_count; return 0; } /* * Handle the channel info event for firmware which only sends one * chan_info event per scanned channel. */ static void ath10k_wmi_event_chan_info_unpaired(struct ath10k *ar, struct chan_info_params *params) { struct survey_info *survey; int idx; if (params->cmd_flags & WMI_CHAN_INFO_FLAG_COMPLETE) { ath10k_dbg(ar, ATH10K_DBG_WMI, "chan info report completed\n"); return; } idx = freq_to_idx(ar, params->freq); if (idx >= ARRAY_SIZE(ar->survey)) { ath10k_warn(ar, "chan info: invalid frequency %d (idx %d out of bounds)\n", params->freq, idx); return; } survey = &ar->survey[idx]; if (!params->mac_clk_mhz) return; memset(survey, 0, sizeof(*survey)); survey->noise = params->noise_floor; survey->time = (params->cycle_count / params->mac_clk_mhz) / 1000; survey->time_busy = (params->rx_clear_count / params->mac_clk_mhz) / 1000; survey->filled |= SURVEY_INFO_NOISE_DBM | SURVEY_INFO_TIME | SURVEY_INFO_TIME_BUSY; } /* * Handle the channel info event for firmware which sends chan_info * event in pairs(start and stop events) for every scanned channel. */ static void ath10k_wmi_event_chan_info_paired(struct ath10k *ar, struct chan_info_params *params) { struct survey_info *survey; int idx; idx = freq_to_idx(ar, params->freq); if (idx >= ARRAY_SIZE(ar->survey)) { ath10k_warn(ar, "chan info: invalid frequency %d (idx %d out of bounds)\n", params->freq, idx); return; } if (params->cmd_flags & WMI_CHAN_INFO_FLAG_COMPLETE) { if (ar->ch_info_can_report_survey) { survey = &ar->survey[idx]; survey->noise = params->noise_floor; survey->filled = SURVEY_INFO_NOISE_DBM; ath10k_hw_fill_survey_time(ar, survey, params->cycle_count, params->rx_clear_count, ar->survey_last_cycle_count, ar->survey_last_rx_clear_count); } ar->ch_info_can_report_survey = false; } else { ar->ch_info_can_report_survey = true; } if (!(params->cmd_flags & WMI_CHAN_INFO_FLAG_PRE_COMPLETE)) { ar->survey_last_rx_clear_count = params->rx_clear_count; ar->survey_last_cycle_count = params->cycle_count; } } void ath10k_wmi_event_chan_info(struct ath10k *ar, struct sk_buff *skb) { struct chan_info_params ch_info_param; struct wmi_ch_info_ev_arg arg = {}; int ret; ret = ath10k_wmi_pull_ch_info(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse chan info event: %d\n", ret); return; } ch_info_param.err_code = __le32_to_cpu(arg.err_code); ch_info_param.freq = __le32_to_cpu(arg.freq); ch_info_param.cmd_flags = __le32_to_cpu(arg.cmd_flags); ch_info_param.noise_floor = __le32_to_cpu(arg.noise_floor); ch_info_param.rx_clear_count = __le32_to_cpu(arg.rx_clear_count); ch_info_param.cycle_count = __le32_to_cpu(arg.cycle_count); ch_info_param.mac_clk_mhz = __le32_to_cpu(arg.mac_clk_mhz); ath10k_dbg(ar, ATH10K_DBG_WMI, "chan info err_code %d freq %d cmd_flags %d noise_floor %d rx_clear_count %d cycle_count %d\n", ch_info_param.err_code, ch_info_param.freq, ch_info_param.cmd_flags, ch_info_param.noise_floor, ch_info_param.rx_clear_count, ch_info_param.cycle_count); spin_lock_bh(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_STARTING: ath10k_dbg(ar, ATH10K_DBG_WMI, "received chan info event without a scan request, ignoring\n"); goto exit; case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: break; } if (test_bit(ATH10K_FW_FEATURE_SINGLE_CHAN_INFO_PER_CHANNEL, ar->running_fw->fw_file.fw_features)) ath10k_wmi_event_chan_info_unpaired(ar, &ch_info_param); else ath10k_wmi_event_chan_info_paired(ar, &ch_info_param); exit: spin_unlock_bh(&ar->data_lock); } void ath10k_wmi_event_echo(struct ath10k *ar, struct sk_buff *skb) { struct wmi_echo_ev_arg arg = {}; int ret; ret = ath10k_wmi_pull_echo_ev(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse echo: %d\n", ret); return; } ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event echo value 0x%08x\n", le32_to_cpu(arg.value)); if (le32_to_cpu(arg.value) == ATH10K_WMI_BARRIER_ECHO_ID) complete(&ar->wmi.barrier); } int ath10k_wmi_event_debug_mesg(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event debug mesg len %d\n", skb->len); trace_ath10k_wmi_dbglog(ar, skb->data, skb->len); return 0; } void ath10k_wmi_pull_pdev_stats_base(const struct wmi_pdev_stats_base *src, struct ath10k_fw_stats_pdev *dst) { dst->ch_noise_floor = __le32_to_cpu(src->chan_nf); dst->tx_frame_count = __le32_to_cpu(src->tx_frame_count); dst->rx_frame_count = __le32_to_cpu(src->rx_frame_count); dst->rx_clear_count = __le32_to_cpu(src->rx_clear_count); dst->cycle_count = __le32_to_cpu(src->cycle_count); dst->phy_err_count = __le32_to_cpu(src->phy_err_count); dst->chan_tx_power = __le32_to_cpu(src->chan_tx_pwr); } void ath10k_wmi_pull_pdev_stats_tx(const struct wmi_pdev_stats_tx *src, struct ath10k_fw_stats_pdev *dst) { dst->comp_queued = __le32_to_cpu(src->comp_queued); dst->comp_delivered = __le32_to_cpu(src->comp_delivered); dst->msdu_enqued = __le32_to_cpu(src->msdu_enqued); dst->mpdu_enqued = __le32_to_cpu(src->mpdu_enqued); dst->wmm_drop = __le32_to_cpu(src->wmm_drop); dst->local_enqued = __le32_to_cpu(src->local_enqued); dst->local_freed = __le32_to_cpu(src->local_freed); dst->hw_queued = __le32_to_cpu(src->hw_queued); dst->hw_reaped = __le32_to_cpu(src->hw_reaped); dst->underrun = __le32_to_cpu(src->underrun); dst->tx_abort = __le32_to_cpu(src->tx_abort); dst->mpdus_requeued = __le32_to_cpu(src->mpdus_requeued); dst->tx_ko = __le32_to_cpu(src->tx_ko); dst->data_rc = __le32_to_cpu(src->data_rc); dst->self_triggers = __le32_to_cpu(src->self_triggers); dst->sw_retry_failure = __le32_to_cpu(src->sw_retry_failure); dst->illgl_rate_phy_err = __le32_to_cpu(src->illgl_rate_phy_err); dst->pdev_cont_xretry = __le32_to_cpu(src->pdev_cont_xretry); dst->pdev_tx_timeout = __le32_to_cpu(src->pdev_tx_timeout); dst->pdev_resets = __le32_to_cpu(src->pdev_resets); dst->phy_underrun = __le32_to_cpu(src->phy_underrun); dst->txop_ovf = __le32_to_cpu(src->txop_ovf); } static void ath10k_wmi_10_4_pull_pdev_stats_tx(const struct wmi_10_4_pdev_stats_tx *src, struct ath10k_fw_stats_pdev *dst) { dst->comp_queued = __le32_to_cpu(src->comp_queued); dst->comp_delivered = __le32_to_cpu(src->comp_delivered); dst->msdu_enqued = __le32_to_cpu(src->msdu_enqued); dst->mpdu_enqued = __le32_to_cpu(src->mpdu_enqued); dst->wmm_drop = __le32_to_cpu(src->wmm_drop); dst->local_enqued = __le32_to_cpu(src->local_enqued); dst->local_freed = __le32_to_cpu(src->local_freed); dst->hw_queued = __le32_to_cpu(src->hw_queued); dst->hw_reaped = __le32_to_cpu(src->hw_reaped); dst->underrun = __le32_to_cpu(src->underrun); dst->tx_abort = __le32_to_cpu(src->tx_abort); dst->mpdus_requeued = __le32_to_cpu(src->mpdus_requeued); dst->tx_ko = __le32_to_cpu(src->tx_ko); dst->data_rc = __le32_to_cpu(src->data_rc); dst->self_triggers = __le32_to_cpu(src->self_triggers); dst->sw_retry_failure = __le32_to_cpu(src->sw_retry_failure); dst->illgl_rate_phy_err = __le32_to_cpu(src->illgl_rate_phy_err); dst->pdev_cont_xretry = __le32_to_cpu(src->pdev_cont_xretry); dst->pdev_tx_timeout = __le32_to_cpu(src->pdev_tx_timeout); dst->pdev_resets = __le32_to_cpu(src->pdev_resets); dst->phy_underrun = __le32_to_cpu(src->phy_underrun); dst->txop_ovf = __le32_to_cpu(src->txop_ovf); dst->hw_paused = __le32_to_cpu(src->hw_paused); dst->seq_posted = __le32_to_cpu(src->seq_posted); dst->seq_failed_queueing = __le32_to_cpu(src->seq_failed_queueing); dst->seq_completed = __le32_to_cpu(src->seq_completed); dst->seq_restarted = __le32_to_cpu(src->seq_restarted); dst->mu_seq_posted = __le32_to_cpu(src->mu_seq_posted); dst->mpdus_sw_flush = __le32_to_cpu(src->mpdus_sw_flush); dst->mpdus_hw_filter = __le32_to_cpu(src->mpdus_hw_filter); dst->mpdus_truncated = __le32_to_cpu(src->mpdus_truncated); dst->mpdus_ack_failed = __le32_to_cpu(src->mpdus_ack_failed); dst->mpdus_hw_filter = __le32_to_cpu(src->mpdus_hw_filter); dst->mpdus_expired = __le32_to_cpu(src->mpdus_expired); } void ath10k_wmi_pull_pdev_stats_rx(const struct wmi_pdev_stats_rx *src, struct ath10k_fw_stats_pdev *dst) { dst->mid_ppdu_route_change = __le32_to_cpu(src->mid_ppdu_route_change); dst->status_rcvd = __le32_to_cpu(src->status_rcvd); dst->r0_frags = __le32_to_cpu(src->r0_frags); dst->r1_frags = __le32_to_cpu(src->r1_frags); dst->r2_frags = __le32_to_cpu(src->r2_frags); dst->r3_frags = __le32_to_cpu(src->r3_frags); dst->htt_msdus = __le32_to_cpu(src->htt_msdus); dst->htt_mpdus = __le32_to_cpu(src->htt_mpdus); dst->loc_msdus = __le32_to_cpu(src->loc_msdus); dst->loc_mpdus = __le32_to_cpu(src->loc_mpdus); dst->oversize_amsdu = __le32_to_cpu(src->oversize_amsdu); dst->phy_errs = __le32_to_cpu(src->phy_errs); dst->phy_err_drop = __le32_to_cpu(src->phy_err_drop); dst->mpdu_errs = __le32_to_cpu(src->mpdu_errs); } void ath10k_wmi_pull_pdev_stats_extra(const struct wmi_pdev_stats_extra *src, struct ath10k_fw_stats_pdev *dst) { dst->ack_rx_bad = __le32_to_cpu(src->ack_rx_bad); dst->rts_bad = __le32_to_cpu(src->rts_bad); dst->rts_good = __le32_to_cpu(src->rts_good); dst->fcs_bad = __le32_to_cpu(src->fcs_bad); dst->no_beacons = __le32_to_cpu(src->no_beacons); dst->mib_int_count = __le32_to_cpu(src->mib_int_count); } void ath10k_wmi_pull_peer_stats(const struct wmi_peer_stats *src, struct ath10k_fw_stats_peer *dst) { ether_addr_copy(dst->peer_macaddr, src->peer_macaddr.addr); dst->peer_rssi = __le32_to_cpu(src->peer_rssi); dst->peer_tx_rate = __le32_to_cpu(src->peer_tx_rate); } static void ath10k_wmi_10_4_pull_peer_stats(const struct wmi_10_4_peer_stats *src, struct ath10k_fw_stats_peer *dst) { ether_addr_copy(dst->peer_macaddr, src->peer_macaddr.addr); dst->peer_rssi = __le32_to_cpu(src->peer_rssi); dst->peer_tx_rate = __le32_to_cpu(src->peer_tx_rate); dst->peer_rx_rate = __le32_to_cpu(src->peer_rx_rate); } static void ath10k_wmi_10_4_pull_vdev_stats(const struct wmi_vdev_stats_extd *src, struct ath10k_fw_stats_vdev_extd *dst) { dst->vdev_id = __le32_to_cpu(src->vdev_id); dst->ppdu_aggr_cnt = __le32_to_cpu(src->ppdu_aggr_cnt); dst->ppdu_noack = __le32_to_cpu(src->ppdu_noack); dst->mpdu_queued = __le32_to_cpu(src->mpdu_queued); dst->ppdu_nonaggr_cnt = __le32_to_cpu(src->ppdu_nonaggr_cnt); dst->mpdu_sw_requeued = __le32_to_cpu(src->mpdu_sw_requeued); dst->mpdu_suc_retry = __le32_to_cpu(src->mpdu_suc_retry); dst->mpdu_suc_multitry = __le32_to_cpu(src->mpdu_suc_multitry); dst->mpdu_fail_retry = __le32_to_cpu(src->mpdu_fail_retry); dst->tx_ftm_suc = __le32_to_cpu(src->tx_ftm_suc); dst->tx_ftm_suc_retry = __le32_to_cpu(src->tx_ftm_suc_retry); dst->tx_ftm_fail = __le32_to_cpu(src->tx_ftm_fail); dst->rx_ftmr_cnt = __le32_to_cpu(src->rx_ftmr_cnt); dst->rx_ftmr_dup_cnt = __le32_to_cpu(src->rx_ftmr_dup_cnt); dst->rx_iftmr_cnt = __le32_to_cpu(src->rx_iftmr_cnt); dst->rx_iftmr_dup_cnt = __le32_to_cpu(src->rx_iftmr_dup_cnt); } static int ath10k_wmi_main_op_pull_fw_stats(struct ath10k *ar, struct sk_buff *skb, struct ath10k_fw_stats *stats) { const struct wmi_stats_event *ev = (void *)skb->data; u32 num_pdev_stats, num_peer_stats; int i; if (!skb_pull(skb, sizeof(*ev))) return -EPROTO; num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats); num_peer_stats = __le32_to_cpu(ev->num_peer_stats); for (i = 0; i < num_pdev_stats; i++) { const struct wmi_pdev_stats *src; struct ath10k_fw_stats_pdev *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_pdev_stats_base(&src->base, dst); ath10k_wmi_pull_pdev_stats_tx(&src->tx, dst); ath10k_wmi_pull_pdev_stats_rx(&src->rx, dst); list_add_tail(&dst->list, &stats->pdevs); } /* fw doesn't implement vdev stats */ for (i = 0; i < num_peer_stats; i++) { const struct wmi_peer_stats *src; struct ath10k_fw_stats_peer *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_peer_stats(src, dst); list_add_tail(&dst->list, &stats->peers); } return 0; } static int ath10k_wmi_10x_op_pull_fw_stats(struct ath10k *ar, struct sk_buff *skb, struct ath10k_fw_stats *stats) { const struct wmi_stats_event *ev = (void *)skb->data; u32 num_pdev_stats, num_peer_stats; int i; if (!skb_pull(skb, sizeof(*ev))) return -EPROTO; num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats); num_peer_stats = __le32_to_cpu(ev->num_peer_stats); for (i = 0; i < num_pdev_stats; i++) { const struct wmi_10x_pdev_stats *src; struct ath10k_fw_stats_pdev *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_pdev_stats_base(&src->base, dst); ath10k_wmi_pull_pdev_stats_tx(&src->tx, dst); ath10k_wmi_pull_pdev_stats_rx(&src->rx, dst); ath10k_wmi_pull_pdev_stats_extra(&src->extra, dst); list_add_tail(&dst->list, &stats->pdevs); } /* fw doesn't implement vdev stats */ for (i = 0; i < num_peer_stats; i++) { const struct wmi_10x_peer_stats *src; struct ath10k_fw_stats_peer *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_peer_stats(&src->old, dst); dst->peer_rx_rate = __le32_to_cpu(src->peer_rx_rate); list_add_tail(&dst->list, &stats->peers); } return 0; } static int ath10k_wmi_10_2_op_pull_fw_stats(struct ath10k *ar, struct sk_buff *skb, struct ath10k_fw_stats *stats) { const struct wmi_10_2_stats_event *ev = (void *)skb->data; u32 num_pdev_stats; u32 num_pdev_ext_stats; u32 num_peer_stats; int i; if (!skb_pull(skb, sizeof(*ev))) return -EPROTO; num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats); num_pdev_ext_stats = __le32_to_cpu(ev->num_pdev_ext_stats); num_peer_stats = __le32_to_cpu(ev->num_peer_stats); for (i = 0; i < num_pdev_stats; i++) { const struct wmi_10_2_pdev_stats *src; struct ath10k_fw_stats_pdev *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_pdev_stats_base(&src->base, dst); ath10k_wmi_pull_pdev_stats_tx(&src->tx, dst); ath10k_wmi_pull_pdev_stats_rx(&src->rx, dst); ath10k_wmi_pull_pdev_stats_extra(&src->extra, dst); /* FIXME: expose 10.2 specific values */ list_add_tail(&dst->list, &stats->pdevs); } for (i = 0; i < num_pdev_ext_stats; i++) { const struct wmi_10_2_pdev_ext_stats *src; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; /* FIXME: expose values to userspace * * Note: Even though this loop seems to do nothing it is * required to parse following sub-structures properly. */ } /* fw doesn't implement vdev stats */ for (i = 0; i < num_peer_stats; i++) { const struct wmi_10_2_peer_stats *src; struct ath10k_fw_stats_peer *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_peer_stats(&src->old, dst); dst->peer_rx_rate = __le32_to_cpu(src->peer_rx_rate); /* FIXME: expose 10.2 specific values */ list_add_tail(&dst->list, &stats->peers); } return 0; } static int ath10k_wmi_10_2_4_op_pull_fw_stats(struct ath10k *ar, struct sk_buff *skb, struct ath10k_fw_stats *stats) { const struct wmi_10_2_stats_event *ev = (void *)skb->data; u32 num_pdev_stats; u32 num_pdev_ext_stats; u32 num_peer_stats; int i; if (!skb_pull(skb, sizeof(*ev))) return -EPROTO; num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats); num_pdev_ext_stats = __le32_to_cpu(ev->num_pdev_ext_stats); num_peer_stats = __le32_to_cpu(ev->num_peer_stats); for (i = 0; i < num_pdev_stats; i++) { const struct wmi_10_2_pdev_stats *src; struct ath10k_fw_stats_pdev *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_pdev_stats_base(&src->base, dst); ath10k_wmi_pull_pdev_stats_tx(&src->tx, dst); ath10k_wmi_pull_pdev_stats_rx(&src->rx, dst); ath10k_wmi_pull_pdev_stats_extra(&src->extra, dst); /* FIXME: expose 10.2 specific values */ list_add_tail(&dst->list, &stats->pdevs); } for (i = 0; i < num_pdev_ext_stats; i++) { const struct wmi_10_2_pdev_ext_stats *src; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; /* FIXME: expose values to userspace * * Note: Even though this loop seems to do nothing it is * required to parse following sub-structures properly. */ } /* fw doesn't implement vdev stats */ for (i = 0; i < num_peer_stats; i++) { const struct wmi_10_2_4_ext_peer_stats *src; struct ath10k_fw_stats_peer *dst; int stats_len; if (test_bit(WMI_SERVICE_PEER_STATS, ar->wmi.svc_map)) stats_len = sizeof(struct wmi_10_2_4_ext_peer_stats); else stats_len = sizeof(struct wmi_10_2_4_peer_stats); src = (void *)skb->data; if (!skb_pull(skb, stats_len)) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_peer_stats(&src->common.old, dst); dst->peer_rx_rate = __le32_to_cpu(src->common.peer_rx_rate); if (ath10k_peer_stats_enabled(ar)) dst->rx_duration = __le32_to_cpu(src->rx_duration); /* FIXME: expose 10.2 specific values */ list_add_tail(&dst->list, &stats->peers); } return 0; } static int ath10k_wmi_10_4_op_pull_fw_stats(struct ath10k *ar, struct sk_buff *skb, struct ath10k_fw_stats *stats) { const struct wmi_10_2_stats_event *ev = (void *)skb->data; u32 num_pdev_stats; u32 num_pdev_ext_stats; u32 num_vdev_stats; u32 num_peer_stats; u32 num_bcnflt_stats; u32 stats_id; int i; if (!skb_pull(skb, sizeof(*ev))) return -EPROTO; num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats); num_pdev_ext_stats = __le32_to_cpu(ev->num_pdev_ext_stats); num_vdev_stats = __le32_to_cpu(ev->num_vdev_stats); num_peer_stats = __le32_to_cpu(ev->num_peer_stats); num_bcnflt_stats = __le32_to_cpu(ev->num_bcnflt_stats); stats_id = __le32_to_cpu(ev->stats_id); for (i = 0; i < num_pdev_stats; i++) { const struct wmi_10_4_pdev_stats *src; struct ath10k_fw_stats_pdev *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_pdev_stats_base(&src->base, dst); ath10k_wmi_10_4_pull_pdev_stats_tx(&src->tx, dst); ath10k_wmi_pull_pdev_stats_rx(&src->rx, dst); dst->rx_ovfl_errs = __le32_to_cpu(src->rx_ovfl_errs); ath10k_wmi_pull_pdev_stats_extra(&src->extra, dst); list_add_tail(&dst->list, &stats->pdevs); } for (i = 0; i < num_pdev_ext_stats; i++) { const struct wmi_10_2_pdev_ext_stats *src; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; /* FIXME: expose values to userspace * * Note: Even though this loop seems to do nothing it is * required to parse following sub-structures properly. */ } for (i = 0; i < num_vdev_stats; i++) { const struct wmi_vdev_stats *src; /* Ignore vdev stats here as it has only vdev id. Actual vdev * stats will be retrieved from vdev extended stats. */ src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; } for (i = 0; i < num_peer_stats; i++) { const struct wmi_10_4_peer_stats *src; struct ath10k_fw_stats_peer *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_10_4_pull_peer_stats(src, dst); list_add_tail(&dst->list, &stats->peers); } for (i = 0; i < num_bcnflt_stats; i++) { const struct wmi_10_4_bss_bcn_filter_stats *src; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; /* FIXME: expose values to userspace * * Note: Even though this loop seems to do nothing it is * required to parse following sub-structures properly. */ } if (stats_id & WMI_10_4_STAT_PEER_EXTD) { stats->extended = true; for (i = 0; i < num_peer_stats; i++) { const struct wmi_10_4_peer_extd_stats *src; struct ath10k_fw_extd_stats_peer *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ether_addr_copy(dst->peer_macaddr, src->peer_macaddr.addr); dst->rx_duration = __le32_to_cpu(src->rx_duration); list_add_tail(&dst->list, &stats->peers_extd); } } if (stats_id & WMI_10_4_STAT_VDEV_EXTD) { for (i = 0; i < num_vdev_stats; i++) { const struct wmi_vdev_stats_extd *src; struct ath10k_fw_stats_vdev_extd *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_10_4_pull_vdev_stats(src, dst); list_add_tail(&dst->list, &stats->vdevs); } } return 0; } void ath10k_wmi_event_update_stats(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_UPDATE_STATS_EVENTID\n"); ath10k_debug_fw_stats_process(ar, skb); } static int ath10k_wmi_op_pull_vdev_start_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_vdev_start_ev_arg *arg) { struct wmi_vdev_start_response_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->vdev_id = ev->vdev_id; arg->req_id = ev->req_id; arg->resp_type = ev->resp_type; arg->status = ev->status; return 0; } void ath10k_wmi_event_vdev_start_resp(struct ath10k *ar, struct sk_buff *skb) { struct wmi_vdev_start_ev_arg arg = {}; int ret; u32 status; ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_VDEV_START_RESP_EVENTID\n"); ar->last_wmi_vdev_start_status = 0; ret = ath10k_wmi_pull_vdev_start(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse vdev start event: %d\n", ret); ar->last_wmi_vdev_start_status = ret; goto out; } status = __le32_to_cpu(arg.status); if (WARN_ON_ONCE(status)) { ath10k_warn(ar, "vdev-start-response reports status error: %d (%s)\n", status, (status == WMI_VDEV_START_CHAN_INVALID) ? "chan-invalid" : "unknown"); /* Setup is done one way or another though, so we should still * do the completion, so don't return here. */ ar->last_wmi_vdev_start_status = -EINVAL; } out: complete(&ar->vdev_setup_done); } void ath10k_wmi_event_vdev_stopped(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_VDEV_STOPPED_EVENTID\n"); complete(&ar->vdev_setup_done); } static int ath10k_wmi_op_pull_peer_kick_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_peer_kick_ev_arg *arg) { struct wmi_peer_sta_kickout_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->mac_addr = ev->peer_macaddr.addr; return 0; } void ath10k_wmi_event_peer_sta_kickout(struct ath10k *ar, struct sk_buff *skb) { struct wmi_peer_kick_ev_arg arg = {}; struct ieee80211_sta *sta; int ret; ret = ath10k_wmi_pull_peer_kick(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse peer kickout event: %d\n", ret); return; } ath10k_dbg(ar, ATH10K_DBG_STA, "wmi event peer sta kickout %pM\n", arg.mac_addr); rcu_read_lock(); sta = ieee80211_find_sta_by_ifaddr(ar->hw, arg.mac_addr, NULL); if (!sta) { ath10k_warn(ar, "Spurious quick kickout for STA %pM\n", arg.mac_addr); goto exit; } ieee80211_report_low_ack(sta, 10); exit: rcu_read_unlock(); } /* * FIXME * * We don't report to mac80211 sleep state of connected * stations. Due to this mac80211 can't fill in TIM IE * correctly. * * I know of no way of getting nullfunc frames that contain * sleep transition from connected stations - these do not * seem to be sent from the target to the host. There also * doesn't seem to be a dedicated event for that. So the * only way left to do this would be to read tim_bitmap * during SWBA. * * We could probably try using tim_bitmap from SWBA to tell * mac80211 which stations are asleep and which are not. The * problem here is calling mac80211 functions so many times * could take too long and make us miss the time to submit * the beacon to the target. * * So as a workaround we try to extend the TIM IE if there * is unicast buffered for stations with aid > 7 and fill it * in ourselves. */ static void ath10k_wmi_update_tim(struct ath10k *ar, struct ath10k_vif *arvif, struct sk_buff *bcn, const struct wmi_tim_info_arg *tim_info) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)bcn->data; struct ieee80211_tim_ie *tim; u8 *ies, *ie; u8 ie_len, pvm_len; __le32 t; u32 v, tim_len; /* When FW reports 0 in tim_len, ensure at least first byte * in tim_bitmap is considered for pvm calculation. */ tim_len = tim_info->tim_len ? __le32_to_cpu(tim_info->tim_len) : 1; /* if next SWBA has no tim_changed the tim_bitmap is garbage. * we must copy the bitmap upon change and reuse it later */ if (__le32_to_cpu(tim_info->tim_changed)) { int i; if (sizeof(arvif->u.ap.tim_bitmap) < tim_len) { ath10k_warn(ar, "SWBA TIM field is too big (%u), truncated it to %zu", tim_len, sizeof(arvif->u.ap.tim_bitmap)); tim_len = sizeof(arvif->u.ap.tim_bitmap); } for (i = 0; i < tim_len; i++) { t = tim_info->tim_bitmap[i / 4]; v = __le32_to_cpu(t); arvif->u.ap.tim_bitmap[i] = (v >> ((i % 4) * 8)) & 0xFF; } /* FW reports either length 0 or length based on max supported * station. so we calculate this on our own */ arvif->u.ap.tim_len = 0; for (i = 0; i < tim_len; i++) if (arvif->u.ap.tim_bitmap[i]) arvif->u.ap.tim_len = i; arvif->u.ap.tim_len++; } ies = bcn->data; ies += ieee80211_hdrlen(hdr->frame_control); ies += 12; /* fixed parameters */ ie = (u8 *)cfg80211_find_ie(WLAN_EID_TIM, ies, (u8 *)skb_tail_pointer(bcn) - ies); if (!ie) { if (arvif->vdev_type != WMI_VDEV_TYPE_IBSS) ath10k_warn(ar, "no tim ie found;\n"); return; } tim = (void *)ie + 2; ie_len = ie[1]; pvm_len = ie_len - 3; /* exclude dtim count, dtim period, bmap ctl */ if (pvm_len < arvif->u.ap.tim_len) { int expand_size = tim_len - pvm_len; int move_size = skb_tail_pointer(bcn) - (ie + 2 + ie_len); void *next_ie = ie + 2 + ie_len; if (skb_put(bcn, expand_size)) { memmove(next_ie + expand_size, next_ie, move_size); ie[1] += expand_size; ie_len += expand_size; pvm_len += expand_size; } else { ath10k_warn(ar, "tim expansion failed\n"); } } if (pvm_len > tim_len) { ath10k_warn(ar, "tim pvm length is too great (%d)\n", pvm_len); return; } tim->bitmap_ctrl = !!__le32_to_cpu(tim_info->tim_mcast); memcpy(tim->virtual_map, arvif->u.ap.tim_bitmap, pvm_len); if (tim->dtim_count == 0) { ATH10K_SKB_CB(bcn)->flags |= ATH10K_SKB_F_DTIM_ZERO; if (__le32_to_cpu(tim_info->tim_mcast) == 1) ATH10K_SKB_CB(bcn)->flags |= ATH10K_SKB_F_DELIVER_CAB; } ath10k_dbg(ar, ATH10K_DBG_MGMT, "dtim %d/%d mcast %d pvmlen %d\n", tim->dtim_count, tim->dtim_period, tim->bitmap_ctrl, pvm_len); } static void ath10k_wmi_update_noa(struct ath10k *ar, struct ath10k_vif *arvif, struct sk_buff *bcn, const struct wmi_p2p_noa_info *noa) { if (!arvif->vif->p2p) return; ath10k_dbg(ar, ATH10K_DBG_MGMT, "noa changed: %d\n", noa->changed); if (noa->changed & WMI_P2P_NOA_CHANGED_BIT) ath10k_p2p_noa_update(arvif, noa); if (arvif->u.ap.noa_data) if (!pskb_expand_head(bcn, 0, arvif->u.ap.noa_len, GFP_ATOMIC)) skb_put_data(bcn, arvif->u.ap.noa_data, arvif->u.ap.noa_len); } static int ath10k_wmi_op_pull_swba_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_swba_ev_arg *arg) { struct wmi_host_swba_event *ev = (void *)skb->data; u32 map; size_t i; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->vdev_map = ev->vdev_map; for (i = 0, map = __le32_to_cpu(ev->vdev_map); map; map >>= 1) { if (!(map & BIT(0))) continue; /* If this happens there were some changes in firmware and * ath10k should update the max size of tim_info array. */ if (WARN_ON_ONCE(i == ARRAY_SIZE(arg->tim_info))) break; if (__le32_to_cpu(ev->bcn_info[i].tim_info.tim_len) > sizeof(ev->bcn_info[i].tim_info.tim_bitmap)) { ath10k_warn(ar, "refusing to parse invalid swba structure\n"); return -EPROTO; } arg->tim_info[i].tim_len = ev->bcn_info[i].tim_info.tim_len; arg->tim_info[i].tim_mcast = ev->bcn_info[i].tim_info.tim_mcast; arg->tim_info[i].tim_bitmap = ev->bcn_info[i].tim_info.tim_bitmap; arg->tim_info[i].tim_changed = ev->bcn_info[i].tim_info.tim_changed; arg->tim_info[i].tim_num_ps_pending = ev->bcn_info[i].tim_info.tim_num_ps_pending; arg->noa_info[i] = &ev->bcn_info[i].p2p_noa_info; i++; } return 0; } static int ath10k_wmi_10_2_4_op_pull_swba_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_swba_ev_arg *arg) { struct wmi_10_2_4_host_swba_event *ev = (void *)skb->data; u32 map; size_t i; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->vdev_map = ev->vdev_map; for (i = 0, map = __le32_to_cpu(ev->vdev_map); map; map >>= 1) { if (!(map & BIT(0))) continue; /* If this happens there were some changes in firmware and * ath10k should update the max size of tim_info array. */ if (WARN_ON_ONCE(i == ARRAY_SIZE(arg->tim_info))) break; if (__le32_to_cpu(ev->bcn_info[i].tim_info.tim_len) > sizeof(ev->bcn_info[i].tim_info.tim_bitmap)) { ath10k_warn(ar, "refusing to parse invalid swba structure\n"); return -EPROTO; } arg->tim_info[i].tim_len = ev->bcn_info[i].tim_info.tim_len; arg->tim_info[i].tim_mcast = ev->bcn_info[i].tim_info.tim_mcast; arg->tim_info[i].tim_bitmap = ev->bcn_info[i].tim_info.tim_bitmap; arg->tim_info[i].tim_changed = ev->bcn_info[i].tim_info.tim_changed; arg->tim_info[i].tim_num_ps_pending = ev->bcn_info[i].tim_info.tim_num_ps_pending; i++; } return 0; } static int ath10k_wmi_10_4_op_pull_swba_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_swba_ev_arg *arg) { struct wmi_10_4_host_swba_event *ev = (void *)skb->data; u32 map, tim_len; size_t i; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->vdev_map = ev->vdev_map; for (i = 0, map = __le32_to_cpu(ev->vdev_map); map; map >>= 1) { if (!(map & BIT(0))) continue; /* If this happens there were some changes in firmware and * ath10k should update the max size of tim_info array. */ if (WARN_ON_ONCE(i == ARRAY_SIZE(arg->tim_info))) break; if (__le32_to_cpu(ev->bcn_info[i].tim_info.tim_len) > sizeof(ev->bcn_info[i].tim_info.tim_bitmap)) { ath10k_warn(ar, "refusing to parse invalid swba structure\n"); return -EPROTO; } tim_len = __le32_to_cpu(ev->bcn_info[i].tim_info.tim_len); if (tim_len) { /* Exclude 4 byte guard length */ tim_len -= 4; arg->tim_info[i].tim_len = __cpu_to_le32(tim_len); } else { arg->tim_info[i].tim_len = 0; } arg->tim_info[i].tim_mcast = ev->bcn_info[i].tim_info.tim_mcast; arg->tim_info[i].tim_bitmap = ev->bcn_info[i].tim_info.tim_bitmap; arg->tim_info[i].tim_changed = ev->bcn_info[i].tim_info.tim_changed; arg->tim_info[i].tim_num_ps_pending = ev->bcn_info[i].tim_info.tim_num_ps_pending; /* 10.4 firmware doesn't have p2p support. notice of absence * info can be ignored for now. */ i++; } return 0; } static enum wmi_txbf_conf ath10k_wmi_10_4_txbf_conf_scheme(struct ath10k *ar) { return WMI_TXBF_CONF_BEFORE_ASSOC; } void ath10k_wmi_event_host_swba(struct ath10k *ar, struct sk_buff *skb) { struct wmi_swba_ev_arg arg = {}; u32 map; int i = -1; const struct wmi_tim_info_arg *tim_info; const struct wmi_p2p_noa_info *noa_info; struct ath10k_vif *arvif; struct sk_buff *bcn; dma_addr_t paddr; int ret, vdev_id = 0; ret = ath10k_wmi_pull_swba(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse swba event: %d\n", ret); return; } map = __le32_to_cpu(arg.vdev_map); ath10k_dbg(ar, ATH10K_DBG_MGMT, "mgmt swba vdev_map 0x%x\n", map); for (; map; map >>= 1, vdev_id++) { if (!(map & 0x1)) continue; i++; if (i >= WMI_MAX_AP_VDEV) { ath10k_warn(ar, "swba has corrupted vdev map\n"); break; } tim_info = &arg.tim_info[i]; noa_info = arg.noa_info[i]; ath10k_dbg(ar, ATH10K_DBG_MGMT, "mgmt event bcn_info %d tim_len %d mcast %d changed %d num_ps_pending %d bitmap 0x%08x%08x%08x%08x\n", i, __le32_to_cpu(tim_info->tim_len), __le32_to_cpu(tim_info->tim_mcast), __le32_to_cpu(tim_info->tim_changed), __le32_to_cpu(tim_info->tim_num_ps_pending), __le32_to_cpu(tim_info->tim_bitmap[3]), __le32_to_cpu(tim_info->tim_bitmap[2]), __le32_to_cpu(tim_info->tim_bitmap[1]), __le32_to_cpu(tim_info->tim_bitmap[0])); /* TODO: Only first 4 word from tim_bitmap is dumped. * Extend debug code to dump full tim_bitmap. */ arvif = ath10k_get_arvif(ar, vdev_id); if (arvif == NULL) { ath10k_warn(ar, "no vif for vdev_id %d found\n", vdev_id); continue; } /* mac80211 would have already asked us to stop beaconing and * bring the vdev down, so continue in that case */ if (!arvif->is_up) continue; /* There are no completions for beacons so wait for next SWBA * before telling mac80211 to decrement CSA counter * * Once CSA counter is completed stop sending beacons until * actual channel switch is done */ if (arvif->vif->bss_conf.csa_active && ieee80211_beacon_cntdwn_is_complete(arvif->vif, 0)) { ieee80211_csa_finish(arvif->vif, 0); continue; } bcn = ieee80211_beacon_get(ar->hw, arvif->vif, 0); if (!bcn) { ath10k_warn(ar, "could not get mac80211 beacon\n"); continue; } ath10k_tx_h_seq_no(arvif->vif, bcn); ath10k_wmi_update_tim(ar, arvif, bcn, tim_info); ath10k_wmi_update_noa(ar, arvif, bcn, noa_info); spin_lock_bh(&ar->data_lock); if (arvif->beacon) { switch (arvif->beacon_state) { case ATH10K_BEACON_SENT: break; case ATH10K_BEACON_SCHEDULED: ath10k_warn(ar, "SWBA overrun on vdev %d, skipped old beacon\n", arvif->vdev_id); break; case ATH10K_BEACON_SENDING: ath10k_warn(ar, "SWBA overrun on vdev %d, skipped new beacon\n", arvif->vdev_id); dev_kfree_skb(bcn); goto skip; } ath10k_mac_vif_beacon_free(arvif); } if (!arvif->beacon_buf) { paddr = dma_map_single(arvif->ar->dev, bcn->data, bcn->len, DMA_TO_DEVICE); ret = dma_mapping_error(arvif->ar->dev, paddr); if (ret) { ath10k_warn(ar, "failed to map beacon: %d\n", ret); dev_kfree_skb_any(bcn); goto skip; } ATH10K_SKB_CB(bcn)->paddr = paddr; } else { if (bcn->len > IEEE80211_MAX_FRAME_LEN) { ath10k_warn(ar, "trimming beacon %d -> %d bytes!\n", bcn->len, IEEE80211_MAX_FRAME_LEN); skb_trim(bcn, IEEE80211_MAX_FRAME_LEN); } memcpy(arvif->beacon_buf, bcn->data, bcn->len); ATH10K_SKB_CB(bcn)->paddr = arvif->beacon_paddr; } arvif->beacon = bcn; arvif->beacon_state = ATH10K_BEACON_SCHEDULED; trace_ath10k_tx_hdr(ar, bcn->data, bcn->len); trace_ath10k_tx_payload(ar, bcn->data, bcn->len); skip: spin_unlock_bh(&ar->data_lock); } ath10k_wmi_tx_beacons_nowait(ar); } void ath10k_wmi_event_tbttoffset_update(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_TBTTOFFSET_UPDATE_EVENTID\n"); } static void ath10k_radar_detected(struct ath10k *ar) { ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs radar detected\n"); ATH10K_DFS_STAT_INC(ar, radar_detected); /* Control radar events reporting in debugfs file * dfs_block_radar_events */ if (ar->dfs_block_radar_events) ath10k_info(ar, "DFS Radar detected, but ignored as requested\n"); else ieee80211_radar_detected(ar->hw, NULL); } static void ath10k_radar_confirmation_work(struct work_struct *work) { struct ath10k *ar = container_of(work, struct ath10k, radar_confirmation_work); struct ath10k_radar_found_info radar_info; int ret, time_left; reinit_completion(&ar->wmi.radar_confirm); spin_lock_bh(&ar->data_lock); memcpy(&radar_info, &ar->last_radar_info, sizeof(radar_info)); spin_unlock_bh(&ar->data_lock); ret = ath10k_wmi_report_radar_found(ar, &radar_info); if (ret) { ath10k_warn(ar, "failed to send radar found %d\n", ret); goto wait_complete; } time_left = wait_for_completion_timeout(&ar->wmi.radar_confirm, ATH10K_WMI_DFS_CONF_TIMEOUT_HZ); if (time_left) { /* DFS Confirmation status event received and * necessary action completed. */ goto wait_complete; } else { /* DFS Confirmation event not received from FW.Considering this * as real radar. */ ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs confirmation not received from fw, considering as radar\n"); goto radar_detected; } radar_detected: ath10k_radar_detected(ar); /* Reset state to allow sending confirmation on consecutive radar * detections, unless radar confirmation is disabled/stopped. */ wait_complete: spin_lock_bh(&ar->data_lock); if (ar->radar_conf_state != ATH10K_RADAR_CONFIRMATION_STOPPED) ar->radar_conf_state = ATH10K_RADAR_CONFIRMATION_IDLE; spin_unlock_bh(&ar->data_lock); } static void ath10k_dfs_radar_report(struct ath10k *ar, struct wmi_phyerr_ev_arg *phyerr, const struct phyerr_radar_report *rr, u64 tsf) { u32 reg0, reg1, tsf32l; struct ieee80211_channel *ch; struct pulse_event pe; struct radar_detector_specs rs; u64 tsf64; u8 rssi, width; struct ath10k_radar_found_info *radar_info; reg0 = __le32_to_cpu(rr->reg0); reg1 = __le32_to_cpu(rr->reg1); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi phyerr radar report chirp %d max_width %d agc_total_gain %d pulse_delta_diff %d\n", MS(reg0, RADAR_REPORT_REG0_PULSE_IS_CHIRP), MS(reg0, RADAR_REPORT_REG0_PULSE_IS_MAX_WIDTH), MS(reg0, RADAR_REPORT_REG0_AGC_TOTAL_GAIN), MS(reg0, RADAR_REPORT_REG0_PULSE_DELTA_DIFF)); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi phyerr radar report pulse_delta_pean %d pulse_sidx %d fft_valid %d agc_mb_gain %d subchan_mask %d\n", MS(reg0, RADAR_REPORT_REG0_PULSE_DELTA_PEAK), MS(reg0, RADAR_REPORT_REG0_PULSE_SIDX), MS(reg1, RADAR_REPORT_REG1_PULSE_SRCH_FFT_VALID), MS(reg1, RADAR_REPORT_REG1_PULSE_AGC_MB_GAIN), MS(reg1, RADAR_REPORT_REG1_PULSE_SUBCHAN_MASK)); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi phyerr radar report pulse_tsf_offset 0x%X pulse_dur: %d\n", MS(reg1, RADAR_REPORT_REG1_PULSE_TSF_OFFSET), MS(reg1, RADAR_REPORT_REG1_PULSE_DUR)); if (!ar->dfs_detector) return; spin_lock_bh(&ar->data_lock); ch = ar->rx_channel; /* fetch target operating channel during channel change */ if (!ch) ch = ar->tgt_oper_chan; spin_unlock_bh(&ar->data_lock); if (!ch) { ath10k_warn(ar, "failed to derive channel for radar pulse, treating as radar\n"); goto radar_detected; } /* report event to DFS pattern detector */ tsf32l = phyerr->tsf_timestamp; tsf64 = tsf & (~0xFFFFFFFFULL); tsf64 |= tsf32l; width = MS(reg1, RADAR_REPORT_REG1_PULSE_DUR); rssi = phyerr->rssi_combined; /* hardware store this as 8 bit signed value, * set to zero if negative number */ if (rssi & 0x80) rssi = 0; pe.ts = tsf64; pe.freq = ch->center_freq; pe.width = width; pe.rssi = rssi; pe.chirp = (MS(reg0, RADAR_REPORT_REG0_PULSE_IS_CHIRP) != 0); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs add pulse freq: %d, width: %d, rssi %d, tsf: %llX\n", pe.freq, pe.width, pe.rssi, pe.ts); ATH10K_DFS_STAT_INC(ar, pulses_detected); if (!ar->dfs_detector->add_pulse(ar->dfs_detector, &pe, &rs)) { ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs no pulse pattern detected, yet\n"); return; } if ((test_bit(WMI_SERVICE_HOST_DFS_CHECK_SUPPORT, ar->wmi.svc_map)) && ar->dfs_detector->region == NL80211_DFS_FCC) { /* Consecutive radar indications need not be * sent to the firmware until we get confirmation * for the previous detected radar. */ spin_lock_bh(&ar->data_lock); if (ar->radar_conf_state != ATH10K_RADAR_CONFIRMATION_IDLE) { spin_unlock_bh(&ar->data_lock); return; } ar->radar_conf_state = ATH10K_RADAR_CONFIRMATION_INPROGRESS; radar_info = &ar->last_radar_info; radar_info->pri_min = rs.pri_min; radar_info->pri_max = rs.pri_max; radar_info->width_min = rs.width_min; radar_info->width_max = rs.width_max; /*TODO Find sidx_min and sidx_max */ radar_info->sidx_min = MS(reg0, RADAR_REPORT_REG0_PULSE_SIDX); radar_info->sidx_max = MS(reg0, RADAR_REPORT_REG0_PULSE_SIDX); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "sending wmi radar found cmd pri_min %d pri_max %d width_min %d width_max %d sidx_min %d sidx_max %d\n", radar_info->pri_min, radar_info->pri_max, radar_info->width_min, radar_info->width_max, radar_info->sidx_min, radar_info->sidx_max); ieee80211_queue_work(ar->hw, &ar->radar_confirmation_work); spin_unlock_bh(&ar->data_lock); return; } radar_detected: ath10k_radar_detected(ar); } static int ath10k_dfs_fft_report(struct ath10k *ar, struct wmi_phyerr_ev_arg *phyerr, const struct phyerr_fft_report *fftr, u64 tsf) { u32 reg0, reg1; u8 rssi, peak_mag; reg0 = __le32_to_cpu(fftr->reg0); reg1 = __le32_to_cpu(fftr->reg1); rssi = phyerr->rssi_combined; ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi phyerr fft report total_gain_db %d base_pwr_db %d fft_chn_idx %d peak_sidx %d\n", MS(reg0, SEARCH_FFT_REPORT_REG0_TOTAL_GAIN_DB), MS(reg0, SEARCH_FFT_REPORT_REG0_BASE_PWR_DB), MS(reg0, SEARCH_FFT_REPORT_REG0_FFT_CHN_IDX), MS(reg0, SEARCH_FFT_REPORT_REG0_PEAK_SIDX)); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi phyerr fft report rel_pwr_db %d avgpwr_db %d peak_mag %d num_store_bin %d\n", MS(reg1, SEARCH_FFT_REPORT_REG1_RELPWR_DB), MS(reg1, SEARCH_FFT_REPORT_REG1_AVGPWR_DB), MS(reg1, SEARCH_FFT_REPORT_REG1_PEAK_MAG), MS(reg1, SEARCH_FFT_REPORT_REG1_NUM_STR_BINS_IB)); peak_mag = MS(reg1, SEARCH_FFT_REPORT_REG1_PEAK_MAG); /* false event detection */ if (rssi == DFS_RSSI_POSSIBLY_FALSE && peak_mag < 2 * DFS_PEAK_MAG_THOLD_POSSIBLY_FALSE) { ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs false pulse detected\n"); ATH10K_DFS_STAT_INC(ar, pulses_discarded); return -EINVAL; } return 0; } void ath10k_wmi_event_dfs(struct ath10k *ar, struct wmi_phyerr_ev_arg *phyerr, u64 tsf) { int buf_len, tlv_len, res, i = 0; const struct phyerr_tlv *tlv; const struct phyerr_radar_report *rr; const struct phyerr_fft_report *fftr; const u8 *tlv_buf; buf_len = phyerr->buf_len; ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi event dfs err_code %d rssi %d tsfl 0x%X tsf64 0x%llX len %d\n", phyerr->phy_err_code, phyerr->rssi_combined, phyerr->tsf_timestamp, tsf, buf_len); /* Skip event if DFS disabled */ if (!IS_ENABLED(CONFIG_ATH10K_DFS_CERTIFIED)) return; ATH10K_DFS_STAT_INC(ar, pulses_total); while (i < buf_len) { if (i + sizeof(*tlv) > buf_len) { ath10k_warn(ar, "too short buf for tlv header (%d)\n", i); return; } tlv = (struct phyerr_tlv *)&phyerr->buf[i]; tlv_len = __le16_to_cpu(tlv->len); tlv_buf = &phyerr->buf[i + sizeof(*tlv)]; ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi event dfs tlv_len %d tlv_tag 0x%02X tlv_sig 0x%02X\n", tlv_len, tlv->tag, tlv->sig); switch (tlv->tag) { case PHYERR_TLV_TAG_RADAR_PULSE_SUMMARY: if (i + sizeof(*tlv) + sizeof(*rr) > buf_len) { ath10k_warn(ar, "too short radar pulse summary (%d)\n", i); return; } rr = (struct phyerr_radar_report *)tlv_buf; ath10k_dfs_radar_report(ar, phyerr, rr, tsf); break; case PHYERR_TLV_TAG_SEARCH_FFT_REPORT: if (i + sizeof(*tlv) + sizeof(*fftr) > buf_len) { ath10k_warn(ar, "too short fft report (%d)\n", i); return; } fftr = (struct phyerr_fft_report *)tlv_buf; res = ath10k_dfs_fft_report(ar, phyerr, fftr, tsf); if (res) return; break; } i += sizeof(*tlv) + tlv_len; } } void ath10k_wmi_event_spectral_scan(struct ath10k *ar, struct wmi_phyerr_ev_arg *phyerr, u64 tsf) { int buf_len, tlv_len, res, i = 0; struct phyerr_tlv *tlv; const void *tlv_buf; const struct phyerr_fft_report *fftr; size_t fftr_len; buf_len = phyerr->buf_len; while (i < buf_len) { if (i + sizeof(*tlv) > buf_len) { ath10k_warn(ar, "failed to parse phyerr tlv header at byte %d\n", i); return; } tlv = (struct phyerr_tlv *)&phyerr->buf[i]; tlv_len = __le16_to_cpu(tlv->len); tlv_buf = &phyerr->buf[i + sizeof(*tlv)]; if (i + sizeof(*tlv) + tlv_len > buf_len) { ath10k_warn(ar, "failed to parse phyerr tlv payload at byte %d\n", i); return; } switch (tlv->tag) { case PHYERR_TLV_TAG_SEARCH_FFT_REPORT: if (sizeof(*fftr) > tlv_len) { ath10k_warn(ar, "failed to parse fft report at byte %d\n", i); return; } fftr_len = tlv_len - sizeof(*fftr); fftr = tlv_buf; res = ath10k_spectral_process_fft(ar, phyerr, fftr, fftr_len, tsf); if (res < 0) { ath10k_dbg(ar, ATH10K_DBG_WMI, "failed to process fft report: %d\n", res); return; } break; } i += sizeof(*tlv) + tlv_len; } } static int ath10k_wmi_op_pull_phyerr_ev_hdr(struct ath10k *ar, struct sk_buff *skb, struct wmi_phyerr_hdr_arg *arg) { struct wmi_phyerr_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; arg->num_phyerrs = __le32_to_cpu(ev->num_phyerrs); arg->tsf_l32 = __le32_to_cpu(ev->tsf_l32); arg->tsf_u32 = __le32_to_cpu(ev->tsf_u32); arg->buf_len = skb->len - sizeof(*ev); arg->phyerrs = ev->phyerrs; return 0; } static int ath10k_wmi_10_4_op_pull_phyerr_ev_hdr(struct ath10k *ar, struct sk_buff *skb, struct wmi_phyerr_hdr_arg *arg) { struct wmi_10_4_phyerr_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; /* 10.4 firmware always reports only one phyerr */ arg->num_phyerrs = 1; arg->tsf_l32 = __le32_to_cpu(ev->tsf_l32); arg->tsf_u32 = __le32_to_cpu(ev->tsf_u32); arg->buf_len = skb->len; arg->phyerrs = skb->data; return 0; } int ath10k_wmi_op_pull_phyerr_ev(struct ath10k *ar, const void *phyerr_buf, int left_len, struct wmi_phyerr_ev_arg *arg) { const struct wmi_phyerr *phyerr = phyerr_buf; int i; if (left_len < sizeof(*phyerr)) { ath10k_warn(ar, "wrong phyerr event head len %d (need: >=%zd)\n", left_len, sizeof(*phyerr)); return -EINVAL; } arg->tsf_timestamp = __le32_to_cpu(phyerr->tsf_timestamp); arg->freq1 = __le16_to_cpu(phyerr->freq1); arg->freq2 = __le16_to_cpu(phyerr->freq2); arg->rssi_combined = phyerr->rssi_combined; arg->chan_width_mhz = phyerr->chan_width_mhz; arg->buf_len = __le32_to_cpu(phyerr->buf_len); arg->buf = phyerr->buf; arg->hdr_len = sizeof(*phyerr); for (i = 0; i < 4; i++) arg->nf_chains[i] = __le16_to_cpu(phyerr->nf_chains[i]); switch (phyerr->phy_err_code) { case PHY_ERROR_GEN_SPECTRAL_SCAN: arg->phy_err_code = PHY_ERROR_SPECTRAL_SCAN; break; case PHY_ERROR_GEN_FALSE_RADAR_EXT: arg->phy_err_code = PHY_ERROR_FALSE_RADAR_EXT; break; case PHY_ERROR_GEN_RADAR: arg->phy_err_code = PHY_ERROR_RADAR; break; default: arg->phy_err_code = PHY_ERROR_UNKNOWN; break; } return 0; } static int ath10k_wmi_10_4_op_pull_phyerr_ev(struct ath10k *ar, const void *phyerr_buf, int left_len, struct wmi_phyerr_ev_arg *arg) { const struct wmi_10_4_phyerr_event *phyerr = phyerr_buf; u32 phy_err_mask; int i; if (left_len < sizeof(*phyerr)) { ath10k_warn(ar, "wrong phyerr event head len %d (need: >=%zd)\n", left_len, sizeof(*phyerr)); return -EINVAL; } arg->tsf_timestamp = __le32_to_cpu(phyerr->tsf_timestamp); arg->freq1 = __le16_to_cpu(phyerr->freq1); arg->freq2 = __le16_to_cpu(phyerr->freq2); arg->rssi_combined = phyerr->rssi_combined; arg->chan_width_mhz = phyerr->chan_width_mhz; arg->buf_len = __le32_to_cpu(phyerr->buf_len); arg->buf = phyerr->buf; arg->hdr_len = sizeof(*phyerr); for (i = 0; i < 4; i++) arg->nf_chains[i] = __le16_to_cpu(phyerr->nf_chains[i]); phy_err_mask = __le32_to_cpu(phyerr->phy_err_mask[0]); if (phy_err_mask & PHY_ERROR_10_4_SPECTRAL_SCAN_MASK) arg->phy_err_code = PHY_ERROR_SPECTRAL_SCAN; else if (phy_err_mask & PHY_ERROR_10_4_RADAR_MASK) arg->phy_err_code = PHY_ERROR_RADAR; else arg->phy_err_code = PHY_ERROR_UNKNOWN; return 0; } void ath10k_wmi_event_phyerr(struct ath10k *ar, struct sk_buff *skb) { struct wmi_phyerr_hdr_arg hdr_arg = {}; struct wmi_phyerr_ev_arg phyerr_arg = {}; const void *phyerr; u32 count, i, buf_len, phy_err_code; u64 tsf; int left_len, ret; ATH10K_DFS_STAT_INC(ar, phy_errors); ret = ath10k_wmi_pull_phyerr_hdr(ar, skb, &hdr_arg); if (ret) { ath10k_warn(ar, "failed to parse phyerr event hdr: %d\n", ret); return; } /* Check number of included events */ count = hdr_arg.num_phyerrs; left_len = hdr_arg.buf_len; tsf = hdr_arg.tsf_u32; tsf <<= 32; tsf |= hdr_arg.tsf_l32; ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event phyerr count %d tsf64 0x%llX\n", count, tsf); phyerr = hdr_arg.phyerrs; for (i = 0; i < count; i++) { ret = ath10k_wmi_pull_phyerr(ar, phyerr, left_len, &phyerr_arg); if (ret) { ath10k_warn(ar, "failed to parse phyerr event (%d)\n", i); return; } left_len -= phyerr_arg.hdr_len; buf_len = phyerr_arg.buf_len; phy_err_code = phyerr_arg.phy_err_code; if (left_len < buf_len) { ath10k_warn(ar, "single event (%d) wrong buf len\n", i); return; } left_len -= buf_len; switch (phy_err_code) { case PHY_ERROR_RADAR: ath10k_wmi_event_dfs(ar, &phyerr_arg, tsf); break; case PHY_ERROR_SPECTRAL_SCAN: ath10k_wmi_event_spectral_scan(ar, &phyerr_arg, tsf); break; case PHY_ERROR_FALSE_RADAR_EXT: ath10k_wmi_event_dfs(ar, &phyerr_arg, tsf); ath10k_wmi_event_spectral_scan(ar, &phyerr_arg, tsf); break; default: break; } phyerr = phyerr + phyerr_arg.hdr_len + buf_len; } } static int ath10k_wmi_10_4_op_pull_dfs_status_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_dfs_status_ev_arg *arg) { struct wmi_dfs_status_ev_arg *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; arg->status = ev->status; return 0; } static void ath10k_wmi_event_dfs_status_check(struct ath10k *ar, struct sk_buff *skb) { struct wmi_dfs_status_ev_arg status_arg = {}; int ret; ret = ath10k_wmi_pull_dfs_status(ar, skb, &status_arg); if (ret) { ath10k_warn(ar, "failed to parse dfs status event: %d\n", ret); return; } ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs status event received from fw: %d\n", status_arg.status); /* Even in case of radar detection failure we follow the same * behaviour as if radar is detected i.e to switch to a different * channel. */ if (status_arg.status == WMI_HW_RADAR_DETECTED || status_arg.status == WMI_RADAR_DETECTION_FAIL) ath10k_radar_detected(ar); complete(&ar->wmi.radar_confirm); } void ath10k_wmi_event_roam(struct ath10k *ar, struct sk_buff *skb) { struct wmi_roam_ev_arg arg = {}; int ret; u32 vdev_id; u32 reason; s32 rssi; ret = ath10k_wmi_pull_roam_ev(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse roam event: %d\n", ret); return; } vdev_id = __le32_to_cpu(arg.vdev_id); reason = __le32_to_cpu(arg.reason); rssi = __le32_to_cpu(arg.rssi); rssi += WMI_SPECTRAL_NOISE_FLOOR_REF_DEFAULT; ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi roam event vdev %u reason 0x%08x rssi %d\n", vdev_id, reason, rssi); if (reason >= WMI_ROAM_REASON_MAX) ath10k_warn(ar, "ignoring unknown roam event reason %d on vdev %i\n", reason, vdev_id); switch (reason) { case WMI_ROAM_REASON_BEACON_MISS: ath10k_mac_handle_beacon_miss(ar, vdev_id); break; case WMI_ROAM_REASON_BETTER_AP: case WMI_ROAM_REASON_LOW_RSSI: case WMI_ROAM_REASON_SUITABLE_AP_FOUND: case WMI_ROAM_REASON_HO_FAILED: ath10k_warn(ar, "ignoring not implemented roam event reason %d on vdev %i\n", reason, vdev_id); break; } } void ath10k_wmi_event_profile_match(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_PROFILE_MATCH\n"); } void ath10k_wmi_event_debug_print(struct ath10k *ar, struct sk_buff *skb) { char buf[101], c; int i; for (i = 0; i < sizeof(buf) - 1; i++) { if (i >= skb->len) break; c = skb->data[i]; if (c == '\0') break; if (isascii(c) && isprint(c)) buf[i] = c; else buf[i] = '.'; } if (i == sizeof(buf) - 1) ath10k_warn(ar, "wmi debug print truncated: %d\n", skb->len); /* for some reason the debug prints end with \n, remove that */ if (skb->data[i - 1] == '\n') i--; /* the last byte is always reserved for the null character */ buf[i] = '\0'; ath10k_dbg(ar, ATH10K_DBG_WMI_PRINT, "wmi print '%s'\n", buf); } void ath10k_wmi_event_pdev_qvit(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_PDEV_QVIT_EVENTID\n"); } void ath10k_wmi_event_wlan_profile_data(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_WLAN_PROFILE_DATA_EVENTID\n"); } void ath10k_wmi_event_rtt_measurement_report(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_RTT_MEASUREMENT_REPORT_EVENTID\n"); } void ath10k_wmi_event_tsf_measurement_report(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_TSF_MEASUREMENT_REPORT_EVENTID\n"); } void ath10k_wmi_event_rtt_error_report(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_RTT_ERROR_REPORT_EVENTID\n"); } void ath10k_wmi_event_wow_wakeup_host(struct ath10k *ar, struct sk_buff *skb) { struct wmi_wow_ev_arg ev = {}; int ret; complete(&ar->wow.wakeup_completed); ret = ath10k_wmi_pull_wow_event(ar, skb, &ev); if (ret) { ath10k_warn(ar, "failed to parse wow wakeup event: %d\n", ret); return; } ath10k_dbg(ar, ATH10K_DBG_WMI, "wow wakeup host reason %s\n", wow_reason(ev.wake_reason)); } void ath10k_wmi_event_dcs_interference(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_DCS_INTERFERENCE_EVENTID\n"); } static u8 ath10k_tpc_config_get_rate(struct ath10k *ar, struct wmi_pdev_tpc_config_event *ev, u32 rate_idx, u32 num_chains, u32 rate_code, u8 type) { u8 tpc, num_streams, preamble, ch, stm_idx; num_streams = ATH10K_HW_NSS(rate_code); preamble = ATH10K_HW_PREAMBLE(rate_code); ch = num_chains - 1; tpc = min_t(u8, ev->rates_array[rate_idx], ev->max_reg_allow_pow[ch]); if (__le32_to_cpu(ev->num_tx_chain) <= 1) goto out; if (preamble == WMI_RATE_PREAMBLE_CCK) goto out; stm_idx = num_streams - 1; if (num_chains <= num_streams) goto out; switch (type) { case WMI_TPC_TABLE_TYPE_STBC: tpc = min_t(u8, tpc, ev->max_reg_allow_pow_agstbc[ch - 1][stm_idx]); break; case WMI_TPC_TABLE_TYPE_TXBF: tpc = min_t(u8, tpc, ev->max_reg_allow_pow_agtxbf[ch - 1][stm_idx]); break; case WMI_TPC_TABLE_TYPE_CDD: tpc = min_t(u8, tpc, ev->max_reg_allow_pow_agcdd[ch - 1][stm_idx]); break; default: ath10k_warn(ar, "unknown wmi tpc table type: %d\n", type); tpc = 0; break; } out: return tpc; } static void ath10k_tpc_config_disp_tables(struct ath10k *ar, struct wmi_pdev_tpc_config_event *ev, struct ath10k_tpc_stats *tpc_stats, u8 *rate_code, u16 *pream_table, u8 type) { u32 i, j, pream_idx, flags; u8 tpc[WMI_TPC_TX_N_CHAIN]; char tpc_value[WMI_TPC_TX_N_CHAIN * WMI_TPC_BUF_SIZE]; char buff[WMI_TPC_BUF_SIZE]; flags = __le32_to_cpu(ev->flags); switch (type) { case WMI_TPC_TABLE_TYPE_CDD: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_CDD)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "CDD not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; case WMI_TPC_TABLE_TYPE_STBC: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_STBC)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "STBC not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; case WMI_TPC_TABLE_TYPE_TXBF: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_TXBF)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "TXBF not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; default: ath10k_dbg(ar, ATH10K_DBG_WMI, "invalid table type in wmi tpc event: %d\n", type); return; } pream_idx = 0; for (i = 0; i < tpc_stats->rate_max; i++) { memset(tpc_value, 0, sizeof(tpc_value)); memset(buff, 0, sizeof(buff)); if (i == pream_table[pream_idx]) pream_idx++; for (j = 0; j < tpc_stats->num_tx_chain; j++) { tpc[j] = ath10k_tpc_config_get_rate(ar, ev, i, j + 1, rate_code[i], type); snprintf(buff, sizeof(buff), "%8d ", tpc[j]); strlcat(tpc_value, buff, sizeof(tpc_value)); } tpc_stats->tpc_table[type].pream_idx[i] = pream_idx; tpc_stats->tpc_table[type].rate_code[i] = rate_code[i]; memcpy(tpc_stats->tpc_table[type].tpc_value[i], tpc_value, sizeof(tpc_value)); } } void ath10k_wmi_tpc_config_get_rate_code(u8 *rate_code, u16 *pream_table, u32 num_tx_chain) { u32 i, j, pream_idx; u8 rate_idx; /* Create the rate code table based on the chains supported */ rate_idx = 0; pream_idx = 0; /* Fill CCK rate code */ for (i = 0; i < 4; i++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(i, 0, WMI_RATE_PREAMBLE_CCK); rate_idx++; } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill OFDM rate code */ for (i = 0; i < 8; i++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(i, 0, WMI_RATE_PREAMBLE_OFDM); rate_idx++; } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill HT20 rate code */ for (i = 0; i < num_tx_chain; i++) { for (j = 0; j < 8; j++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(j, i, WMI_RATE_PREAMBLE_HT); rate_idx++; } } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill HT40 rate code */ for (i = 0; i < num_tx_chain; i++) { for (j = 0; j < 8; j++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(j, i, WMI_RATE_PREAMBLE_HT); rate_idx++; } } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill VHT20 rate code */ for (i = 0; i < num_tx_chain; i++) { for (j = 0; j < 10; j++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(j, i, WMI_RATE_PREAMBLE_VHT); rate_idx++; } } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill VHT40 rate code */ for (i = 0; i < num_tx_chain; i++) { for (j = 0; j < 10; j++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(j, i, WMI_RATE_PREAMBLE_VHT); rate_idx++; } } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill VHT80 rate code */ for (i = 0; i < num_tx_chain; i++) { for (j = 0; j < 10; j++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(j, i, WMI_RATE_PREAMBLE_VHT); rate_idx++; } } pream_table[pream_idx] = rate_idx; pream_idx++; rate_code[rate_idx++] = ATH10K_HW_RATECODE(0, 0, WMI_RATE_PREAMBLE_CCK); rate_code[rate_idx++] = ATH10K_HW_RATECODE(0, 0, WMI_RATE_PREAMBLE_OFDM); rate_code[rate_idx++] = ATH10K_HW_RATECODE(0, 0, WMI_RATE_PREAMBLE_CCK); rate_code[rate_idx++] = ATH10K_HW_RATECODE(0, 0, WMI_RATE_PREAMBLE_OFDM); rate_code[rate_idx++] = ATH10K_HW_RATECODE(0, 0, WMI_RATE_PREAMBLE_OFDM); pream_table[pream_idx] = ATH10K_TPC_PREAM_TABLE_END; } void ath10k_wmi_event_pdev_tpc_config(struct ath10k *ar, struct sk_buff *skb) { u32 num_tx_chain, rate_max; u8 rate_code[WMI_TPC_RATE_MAX]; u16 pream_table[WMI_TPC_PREAM_TABLE_MAX]; struct wmi_pdev_tpc_config_event *ev; struct ath10k_tpc_stats *tpc_stats; ev = (struct wmi_pdev_tpc_config_event *)skb->data; num_tx_chain = __le32_to_cpu(ev->num_tx_chain); if (num_tx_chain > WMI_TPC_TX_N_CHAIN) { ath10k_warn(ar, "number of tx chain is %d greater than TPC configured tx chain %d\n", num_tx_chain, WMI_TPC_TX_N_CHAIN); return; } rate_max = __le32_to_cpu(ev->rate_max); if (rate_max > WMI_TPC_RATE_MAX) { ath10k_warn(ar, "number of rate is %d greater than TPC configured rate %d\n", rate_max, WMI_TPC_RATE_MAX); rate_max = WMI_TPC_RATE_MAX; } tpc_stats = kzalloc(sizeof(*tpc_stats), GFP_ATOMIC); if (!tpc_stats) return; ath10k_wmi_tpc_config_get_rate_code(rate_code, pream_table, num_tx_chain); tpc_stats->chan_freq = __le32_to_cpu(ev->chan_freq); tpc_stats->phy_mode = __le32_to_cpu(ev->phy_mode); tpc_stats->ctl = __le32_to_cpu(ev->ctl); tpc_stats->reg_domain = __le32_to_cpu(ev->reg_domain); tpc_stats->twice_antenna_gain = a_sle32_to_cpu(ev->twice_antenna_gain); tpc_stats->twice_antenna_reduction = __le32_to_cpu(ev->twice_antenna_reduction); tpc_stats->power_limit = __le32_to_cpu(ev->power_limit); tpc_stats->twice_max_rd_power = __le32_to_cpu(ev->twice_max_rd_power); tpc_stats->num_tx_chain = num_tx_chain; tpc_stats->rate_max = rate_max; ath10k_tpc_config_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_CDD); ath10k_tpc_config_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_STBC); ath10k_tpc_config_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_TXBF); ath10k_debug_tpc_stats_process(ar, tpc_stats); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event tpc config channel %d mode %d ctl %d regd %d gain %d %d limit %d max_power %d tx_chanins %d rates %d\n", __le32_to_cpu(ev->chan_freq), __le32_to_cpu(ev->phy_mode), __le32_to_cpu(ev->ctl), __le32_to_cpu(ev->reg_domain), a_sle32_to_cpu(ev->twice_antenna_gain), __le32_to_cpu(ev->twice_antenna_reduction), __le32_to_cpu(ev->power_limit), __le32_to_cpu(ev->twice_max_rd_power) / 2, __le32_to_cpu(ev->num_tx_chain), __le32_to_cpu(ev->rate_max)); } static u8 ath10k_wmi_tpc_final_get_rate(struct ath10k *ar, struct wmi_pdev_tpc_final_table_event *ev, u32 rate_idx, u32 num_chains, u32 rate_code, u8 type, u32 pream_idx) { u8 tpc, num_streams, preamble, ch, stm_idx; s8 pow_agcdd, pow_agstbc, pow_agtxbf; int pream; num_streams = ATH10K_HW_NSS(rate_code); preamble = ATH10K_HW_PREAMBLE(rate_code); ch = num_chains - 1; stm_idx = num_streams - 1; pream = -1; if (__le32_to_cpu(ev->chan_freq) <= 2483) { switch (pream_idx) { case WMI_TPC_PREAM_2GHZ_CCK: pream = 0; break; case WMI_TPC_PREAM_2GHZ_OFDM: pream = 1; break; case WMI_TPC_PREAM_2GHZ_HT20: case WMI_TPC_PREAM_2GHZ_VHT20: pream = 2; break; case WMI_TPC_PREAM_2GHZ_HT40: case WMI_TPC_PREAM_2GHZ_VHT40: pream = 3; break; case WMI_TPC_PREAM_2GHZ_VHT80: pream = 4; break; default: pream = -1; break; } } if (__le32_to_cpu(ev->chan_freq) >= 5180) { switch (pream_idx) { case WMI_TPC_PREAM_5GHZ_OFDM: pream = 0; break; case WMI_TPC_PREAM_5GHZ_HT20: case WMI_TPC_PREAM_5GHZ_VHT20: pream = 1; break; case WMI_TPC_PREAM_5GHZ_HT40: case WMI_TPC_PREAM_5GHZ_VHT40: pream = 2; break; case WMI_TPC_PREAM_5GHZ_VHT80: pream = 3; break; case WMI_TPC_PREAM_5GHZ_HTCUP: pream = 4; break; default: pream = -1; break; } } if (pream == -1) { ath10k_warn(ar, "unknown wmi tpc final index and frequency: %u, %u\n", pream_idx, __le32_to_cpu(ev->chan_freq)); tpc = 0; goto out; } if (pream == 4) tpc = min_t(u8, ev->rates_array[rate_idx], ev->max_reg_allow_pow[ch]); else tpc = min_t(u8, min_t(u8, ev->rates_array[rate_idx], ev->max_reg_allow_pow[ch]), ev->ctl_power_table[0][pream][stm_idx]); if (__le32_to_cpu(ev->num_tx_chain) <= 1) goto out; if (preamble == WMI_RATE_PREAMBLE_CCK) goto out; if (num_chains <= num_streams) goto out; switch (type) { case WMI_TPC_TABLE_TYPE_STBC: pow_agstbc = ev->max_reg_allow_pow_agstbc[ch - 1][stm_idx]; if (pream == 4) tpc = min_t(u8, tpc, pow_agstbc); else tpc = min_t(u8, min_t(u8, tpc, pow_agstbc), ev->ctl_power_table[0][pream][stm_idx]); break; case WMI_TPC_TABLE_TYPE_TXBF: pow_agtxbf = ev->max_reg_allow_pow_agtxbf[ch - 1][stm_idx]; if (pream == 4) tpc = min_t(u8, tpc, pow_agtxbf); else tpc = min_t(u8, min_t(u8, tpc, pow_agtxbf), ev->ctl_power_table[1][pream][stm_idx]); break; case WMI_TPC_TABLE_TYPE_CDD: pow_agcdd = ev->max_reg_allow_pow_agcdd[ch - 1][stm_idx]; if (pream == 4) tpc = min_t(u8, tpc, pow_agcdd); else tpc = min_t(u8, min_t(u8, tpc, pow_agcdd), ev->ctl_power_table[0][pream][stm_idx]); break; default: ath10k_warn(ar, "unknown wmi tpc final table type: %d\n", type); tpc = 0; break; } out: return tpc; } static void ath10k_wmi_tpc_stats_final_disp_tables(struct ath10k *ar, struct wmi_pdev_tpc_final_table_event *ev, struct ath10k_tpc_stats_final *tpc_stats, u8 *rate_code, u16 *pream_table, u8 type) { u32 i, j, pream_idx, flags; u8 tpc[WMI_TPC_TX_N_CHAIN]; char tpc_value[WMI_TPC_TX_N_CHAIN * WMI_TPC_BUF_SIZE]; char buff[WMI_TPC_BUF_SIZE]; flags = __le32_to_cpu(ev->flags); switch (type) { case WMI_TPC_TABLE_TYPE_CDD: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_CDD)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "CDD not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; case WMI_TPC_TABLE_TYPE_STBC: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_STBC)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "STBC not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; case WMI_TPC_TABLE_TYPE_TXBF: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_TXBF)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "TXBF not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; default: ath10k_dbg(ar, ATH10K_DBG_WMI, "invalid table type in wmi tpc event: %d\n", type); return; } pream_idx = 0; for (i = 0; i < tpc_stats->rate_max; i++) { memset(tpc_value, 0, sizeof(tpc_value)); memset(buff, 0, sizeof(buff)); if (i == pream_table[pream_idx]) pream_idx++; for (j = 0; j < tpc_stats->num_tx_chain; j++) { tpc[j] = ath10k_wmi_tpc_final_get_rate(ar, ev, i, j + 1, rate_code[i], type, pream_idx); snprintf(buff, sizeof(buff), "%8d ", tpc[j]); strlcat(tpc_value, buff, sizeof(tpc_value)); } tpc_stats->tpc_table_final[type].pream_idx[i] = pream_idx; tpc_stats->tpc_table_final[type].rate_code[i] = rate_code[i]; memcpy(tpc_stats->tpc_table_final[type].tpc_value[i], tpc_value, sizeof(tpc_value)); } } void ath10k_wmi_event_tpc_final_table(struct ath10k *ar, struct sk_buff *skb) { u32 num_tx_chain, rate_max; u8 rate_code[WMI_TPC_FINAL_RATE_MAX]; u16 pream_table[WMI_TPC_PREAM_TABLE_MAX]; struct wmi_pdev_tpc_final_table_event *ev; struct ath10k_tpc_stats_final *tpc_stats; ev = (struct wmi_pdev_tpc_final_table_event *)skb->data; num_tx_chain = __le32_to_cpu(ev->num_tx_chain); if (num_tx_chain > WMI_TPC_TX_N_CHAIN) { ath10k_warn(ar, "number of tx chain is %d greater than TPC final configured tx chain %d\n", num_tx_chain, WMI_TPC_TX_N_CHAIN); return; } rate_max = __le32_to_cpu(ev->rate_max); if (rate_max > WMI_TPC_FINAL_RATE_MAX) { ath10k_warn(ar, "number of rate is %d greater than TPC final configured rate %d\n", rate_max, WMI_TPC_FINAL_RATE_MAX); rate_max = WMI_TPC_FINAL_RATE_MAX; } tpc_stats = kzalloc(sizeof(*tpc_stats), GFP_ATOMIC); if (!tpc_stats) return; ath10k_wmi_tpc_config_get_rate_code(rate_code, pream_table, num_tx_chain); tpc_stats->chan_freq = __le32_to_cpu(ev->chan_freq); tpc_stats->phy_mode = __le32_to_cpu(ev->phy_mode); tpc_stats->ctl = __le32_to_cpu(ev->ctl); tpc_stats->reg_domain = __le32_to_cpu(ev->reg_domain); tpc_stats->twice_antenna_gain = a_sle32_to_cpu(ev->twice_antenna_gain); tpc_stats->twice_antenna_reduction = __le32_to_cpu(ev->twice_antenna_reduction); tpc_stats->power_limit = __le32_to_cpu(ev->power_limit); tpc_stats->twice_max_rd_power = __le32_to_cpu(ev->twice_max_rd_power); tpc_stats->num_tx_chain = num_tx_chain; tpc_stats->rate_max = rate_max; ath10k_wmi_tpc_stats_final_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_CDD); ath10k_wmi_tpc_stats_final_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_STBC); ath10k_wmi_tpc_stats_final_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_TXBF); ath10k_debug_tpc_stats_final_process(ar, tpc_stats); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event tpc final table channel %d mode %d ctl %d regd %d gain %d %d limit %d max_power %d tx_chanins %d rates %d\n", __le32_to_cpu(ev->chan_freq), __le32_to_cpu(ev->phy_mode), __le32_to_cpu(ev->ctl), __le32_to_cpu(ev->reg_domain), a_sle32_to_cpu(ev->twice_antenna_gain), __le32_to_cpu(ev->twice_antenna_reduction), __le32_to_cpu(ev->power_limit), __le32_to_cpu(ev->twice_max_rd_power) / 2, __le32_to_cpu(ev->num_tx_chain), __le32_to_cpu(ev->rate_max)); } static void ath10k_wmi_handle_tdls_peer_event(struct ath10k *ar, struct sk_buff *skb) { struct wmi_tdls_peer_event *ev; struct ath10k_peer *peer; struct ath10k_vif *arvif; int vdev_id; int peer_status; int peer_reason; u8 reason; if (skb->len < sizeof(*ev)) { ath10k_err(ar, "received tdls peer event with invalid size (%d bytes)\n", skb->len); return; } ev = (struct wmi_tdls_peer_event *)skb->data; vdev_id = __le32_to_cpu(ev->vdev_id); peer_status = __le32_to_cpu(ev->peer_status); peer_reason = __le32_to_cpu(ev->peer_reason); spin_lock_bh(&ar->data_lock); peer = ath10k_peer_find(ar, vdev_id, ev->peer_macaddr.addr); spin_unlock_bh(&ar->data_lock); if (!peer) { ath10k_warn(ar, "failed to find peer entry for %pM\n", ev->peer_macaddr.addr); return; } switch (peer_status) { case WMI_TDLS_SHOULD_TEARDOWN: switch (peer_reason) { case WMI_TDLS_TEARDOWN_REASON_PTR_TIMEOUT: case WMI_TDLS_TEARDOWN_REASON_NO_RESPONSE: case WMI_TDLS_TEARDOWN_REASON_RSSI: reason = WLAN_REASON_TDLS_TEARDOWN_UNREACHABLE; break; default: reason = WLAN_REASON_TDLS_TEARDOWN_UNSPECIFIED; break; } arvif = ath10k_get_arvif(ar, vdev_id); if (!arvif) { ath10k_warn(ar, "received tdls peer event for invalid vdev id %u\n", vdev_id); return; } ieee80211_tdls_oper_request(arvif->vif, ev->peer_macaddr.addr, NL80211_TDLS_TEARDOWN, reason, GFP_ATOMIC); ath10k_dbg(ar, ATH10K_DBG_WMI, "received tdls teardown event for peer %pM reason %u\n", ev->peer_macaddr.addr, peer_reason); break; default: ath10k_dbg(ar, ATH10K_DBG_WMI, "received unknown tdls peer event %u\n", peer_status); break; } } static void ath10k_wmi_event_peer_sta_ps_state_chg(struct ath10k *ar, struct sk_buff *skb) { struct wmi_peer_sta_ps_state_chg_event *ev; struct ieee80211_sta *sta; struct ath10k_sta *arsta; u8 peer_addr[ETH_ALEN]; lockdep_assert_held(&ar->data_lock); ev = (struct wmi_peer_sta_ps_state_chg_event *)skb->data; ether_addr_copy(peer_addr, ev->peer_macaddr.addr); rcu_read_lock(); sta = ieee80211_find_sta_by_ifaddr(ar->hw, peer_addr, NULL); if (!sta) { ath10k_warn(ar, "failed to find station entry %pM\n", peer_addr); goto exit; } arsta = (struct ath10k_sta *)sta->drv_priv; arsta->peer_ps_state = __le32_to_cpu(ev->peer_ps_state); exit: rcu_read_unlock(); } void ath10k_wmi_event_pdev_ftm_intg(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_PDEV_FTM_INTG_EVENTID\n"); } void ath10k_wmi_event_gtk_offload_status(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_GTK_OFFLOAD_STATUS_EVENTID\n"); } void ath10k_wmi_event_gtk_rekey_fail(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_GTK_REKEY_FAIL_EVENTID\n"); } void ath10k_wmi_event_delba_complete(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_TX_DELBA_COMPLETE_EVENTID\n"); } void ath10k_wmi_event_addba_complete(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_TX_ADDBA_COMPLETE_EVENTID\n"); } void ath10k_wmi_event_vdev_install_key_complete(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_VDEV_INSTALL_KEY_COMPLETE_EVENTID\n"); } void ath10k_wmi_event_inst_rssi_stats(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_INST_RSSI_STATS_EVENTID\n"); } void ath10k_wmi_event_vdev_standby_req(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_VDEV_STANDBY_REQ_EVENTID\n"); } void ath10k_wmi_event_vdev_resume_req(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_VDEV_RESUME_REQ_EVENTID\n"); } static int ath10k_wmi_alloc_chunk(struct ath10k *ar, u32 req_id, u32 num_units, u32 unit_len) { dma_addr_t paddr; u32 pool_size; int idx = ar->wmi.num_mem_chunks; void *vaddr; pool_size = num_units * round_up(unit_len, 4); vaddr = dma_alloc_coherent(ar->dev, pool_size, &paddr, GFP_KERNEL); if (!vaddr) return -ENOMEM; ar->wmi.mem_chunks[idx].vaddr = vaddr; ar->wmi.mem_chunks[idx].paddr = paddr; ar->wmi.mem_chunks[idx].len = pool_size; ar->wmi.mem_chunks[idx].req_id = req_id; ar->wmi.num_mem_chunks++; return num_units; } static int ath10k_wmi_alloc_host_mem(struct ath10k *ar, u32 req_id, u32 num_units, u32 unit_len) { int ret; while (num_units) { ret = ath10k_wmi_alloc_chunk(ar, req_id, num_units, unit_len); if (ret < 0) return ret; num_units -= ret; } return 0; } static bool ath10k_wmi_is_host_mem_allocated(struct ath10k *ar, const struct wlan_host_mem_req **mem_reqs, u32 num_mem_reqs) { u32 req_id, num_units, unit_size, num_unit_info; u32 pool_size; int i, j; bool found; if (ar->wmi.num_mem_chunks != num_mem_reqs) return false; for (i = 0; i < num_mem_reqs; ++i) { req_id = __le32_to_cpu(mem_reqs[i]->req_id); num_units = __le32_to_cpu(mem_reqs[i]->num_units); unit_size = __le32_to_cpu(mem_reqs[i]->unit_size); num_unit_info = __le32_to_cpu(mem_reqs[i]->num_unit_info); if (num_unit_info & NUM_UNITS_IS_NUM_ACTIVE_PEERS) { if (ar->num_active_peers) num_units = ar->num_active_peers + 1; else num_units = ar->max_num_peers + 1; } else if (num_unit_info & NUM_UNITS_IS_NUM_PEERS) { num_units = ar->max_num_peers + 1; } else if (num_unit_info & NUM_UNITS_IS_NUM_VDEVS) { num_units = ar->max_num_vdevs + 1; } found = false; for (j = 0; j < ar->wmi.num_mem_chunks; j++) { if (ar->wmi.mem_chunks[j].req_id == req_id) { pool_size = num_units * round_up(unit_size, 4); if (ar->wmi.mem_chunks[j].len == pool_size) { found = true; break; } } } if (!found) return false; } return true; } static int ath10k_wmi_main_op_pull_svc_rdy_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_svc_rdy_ev_arg *arg) { struct wmi_service_ready_event *ev; size_t i, n; if (skb->len < sizeof(*ev)) return -EPROTO; ev = (void *)skb->data; skb_pull(skb, sizeof(*ev)); arg->min_tx_power = ev->hw_min_tx_power; arg->max_tx_power = ev->hw_max_tx_power; arg->ht_cap = ev->ht_cap_info; arg->vht_cap = ev->vht_cap_info; arg->vht_supp_mcs = ev->vht_supp_mcs; arg->sw_ver0 = ev->sw_version; arg->sw_ver1 = ev->sw_version_1; arg->phy_capab = ev->phy_capability; arg->num_rf_chains = ev->num_rf_chains; arg->eeprom_rd = ev->hal_reg_capabilities.eeprom_rd; arg->low_2ghz_chan = ev->hal_reg_capabilities.low_2ghz_chan; arg->high_2ghz_chan = ev->hal_reg_capabilities.high_2ghz_chan; arg->low_5ghz_chan = ev->hal_reg_capabilities.low_5ghz_chan; arg->high_5ghz_chan = ev->hal_reg_capabilities.high_5ghz_chan; arg->num_mem_reqs = ev->num_mem_reqs; arg->service_map = ev->wmi_service_bitmap; arg->service_map_len = sizeof(ev->wmi_service_bitmap); n = min_t(size_t, __le32_to_cpu(arg->num_mem_reqs), ARRAY_SIZE(arg->mem_reqs)); for (i = 0; i < n; i++) arg->mem_reqs[i] = &ev->mem_reqs[i]; if (skb->len < __le32_to_cpu(arg->num_mem_reqs) * sizeof(arg->mem_reqs[0])) return -EPROTO; return 0; } static int ath10k_wmi_10x_op_pull_svc_rdy_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_svc_rdy_ev_arg *arg) { struct wmi_10x_service_ready_event *ev; int i, n; if (skb->len < sizeof(*ev)) return -EPROTO; ev = (void *)skb->data; skb_pull(skb, sizeof(*ev)); arg->min_tx_power = ev->hw_min_tx_power; arg->max_tx_power = ev->hw_max_tx_power; arg->ht_cap = ev->ht_cap_info; arg->vht_cap = ev->vht_cap_info; arg->vht_supp_mcs = ev->vht_supp_mcs; arg->sw_ver0 = ev->sw_version; arg->phy_capab = ev->phy_capability; arg->num_rf_chains = ev->num_rf_chains; arg->eeprom_rd = ev->hal_reg_capabilities.eeprom_rd; arg->low_2ghz_chan = ev->hal_reg_capabilities.low_2ghz_chan; arg->high_2ghz_chan = ev->hal_reg_capabilities.high_2ghz_chan; arg->low_5ghz_chan = ev->hal_reg_capabilities.low_5ghz_chan; arg->high_5ghz_chan = ev->hal_reg_capabilities.high_5ghz_chan; arg->num_mem_reqs = ev->num_mem_reqs; arg->service_map = ev->wmi_service_bitmap; arg->service_map_len = sizeof(ev->wmi_service_bitmap); /* Deliberately skipping ev->sys_cap_info as WMI and WMI-TLV have * different values. We would need a translation to handle that, * but as we don't currently need anything from sys_cap_info from * WMI interface (only from WMI-TLV) safest it to skip it. */ n = min_t(size_t, __le32_to_cpu(arg->num_mem_reqs), ARRAY_SIZE(arg->mem_reqs)); for (i = 0; i < n; i++) arg->mem_reqs[i] = &ev->mem_reqs[i]; if (skb->len < __le32_to_cpu(arg->num_mem_reqs) * sizeof(arg->mem_reqs[0])) return -EPROTO; return 0; } static void ath10k_wmi_event_service_ready_work(struct work_struct *work) { struct ath10k *ar = container_of(work, struct ath10k, svc_rdy_work); struct sk_buff *skb = ar->svc_rdy_skb; struct wmi_svc_rdy_ev_arg arg = {}; u32 num_units, req_id, unit_size, num_mem_reqs, num_unit_info, i; int ret; bool allocated; if (!skb) { ath10k_warn(ar, "invalid service ready event skb\n"); return; } ret = ath10k_wmi_pull_svc_rdy(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse service ready: %d\n", ret); return; } ath10k_wmi_map_svc(ar, arg.service_map, ar->wmi.svc_map, arg.service_map_len); ar->hw_min_tx_power = __le32_to_cpu(arg.min_tx_power); ar->hw_max_tx_power = __le32_to_cpu(arg.max_tx_power); ar->ht_cap_info = __le32_to_cpu(arg.ht_cap); ar->vht_cap_info = __le32_to_cpu(arg.vht_cap); ar->vht_supp_mcs = __le32_to_cpu(arg.vht_supp_mcs); ar->fw_version_major = (__le32_to_cpu(arg.sw_ver0) & 0xff000000) >> 24; ar->fw_version_minor = (__le32_to_cpu(arg.sw_ver0) & 0x00ffffff); ar->fw_version_release = (__le32_to_cpu(arg.sw_ver1) & 0xffff0000) >> 16; ar->fw_version_build = (__le32_to_cpu(arg.sw_ver1) & 0x0000ffff); ar->phy_capability = __le32_to_cpu(arg.phy_capab); ar->num_rf_chains = __le32_to_cpu(arg.num_rf_chains); ar->hw_eeprom_rd = __le32_to_cpu(arg.eeprom_rd); ar->low_2ghz_chan = __le32_to_cpu(arg.low_2ghz_chan); ar->high_2ghz_chan = __le32_to_cpu(arg.high_2ghz_chan); ar->low_5ghz_chan = __le32_to_cpu(arg.low_5ghz_chan); ar->high_5ghz_chan = __le32_to_cpu(arg.high_5ghz_chan); ar->sys_cap_info = __le32_to_cpu(arg.sys_cap_info); ath10k_dbg_dump(ar, ATH10K_DBG_WMI, NULL, "wmi svc: ", arg.service_map, arg.service_map_len); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi sys_cap_info 0x%x\n", ar->sys_cap_info); if (ar->num_rf_chains > ar->max_spatial_stream) { ath10k_warn(ar, "hardware advertises support for more spatial streams than it should (%d > %d)\n", ar->num_rf_chains, ar->max_spatial_stream); ar->num_rf_chains = ar->max_spatial_stream; } if (!ar->cfg_tx_chainmask) { ar->cfg_tx_chainmask = (1 << ar->num_rf_chains) - 1; ar->cfg_rx_chainmask = (1 << ar->num_rf_chains) - 1; } if (strlen(ar->hw->wiphy->fw_version) == 0) { snprintf(ar->hw->wiphy->fw_version, sizeof(ar->hw->wiphy->fw_version), "%u.%u.%u.%u", ar->fw_version_major, ar->fw_version_minor, ar->fw_version_release, ar->fw_version_build); } num_mem_reqs = __le32_to_cpu(arg.num_mem_reqs); if (num_mem_reqs > WMI_MAX_MEM_REQS) { ath10k_warn(ar, "requested memory chunks number (%d) exceeds the limit\n", num_mem_reqs); return; } if (test_bit(WMI_SERVICE_PEER_CACHING, ar->wmi.svc_map)) { if (test_bit(ATH10K_FW_FEATURE_PEER_FLOW_CONTROL, ar->running_fw->fw_file.fw_features)) ar->num_active_peers = TARGET_10_4_QCACHE_ACTIVE_PEERS_PFC + ar->max_num_vdevs; else ar->num_active_peers = TARGET_10_4_QCACHE_ACTIVE_PEERS + ar->max_num_vdevs; ar->max_num_peers = TARGET_10_4_NUM_QCACHE_PEERS_MAX + ar->max_num_vdevs; ar->num_tids = ar->num_active_peers * 2; ar->max_num_stations = TARGET_10_4_NUM_QCACHE_PEERS_MAX; } /* TODO: Adjust max peer count for cases like WMI_SERVICE_RATECTRL_CACHE * and WMI_SERVICE_IRAM_TIDS, etc. */ allocated = ath10k_wmi_is_host_mem_allocated(ar, arg.mem_reqs, num_mem_reqs); if (allocated) goto skip_mem_alloc; /* Either this event is received during boot time or there is a change * in memory requirement from firmware when compared to last request. * Free any old memory and do a fresh allocation based on the current * memory requirement. */ ath10k_wmi_free_host_mem(ar); for (i = 0; i < num_mem_reqs; ++i) { req_id = __le32_to_cpu(arg.mem_reqs[i]->req_id); num_units = __le32_to_cpu(arg.mem_reqs[i]->num_units); unit_size = __le32_to_cpu(arg.mem_reqs[i]->unit_size); num_unit_info = __le32_to_cpu(arg.mem_reqs[i]->num_unit_info); if (num_unit_info & NUM_UNITS_IS_NUM_ACTIVE_PEERS) { if (ar->num_active_peers) num_units = ar->num_active_peers + 1; else num_units = ar->max_num_peers + 1; } else if (num_unit_info & NUM_UNITS_IS_NUM_PEERS) { /* number of units to allocate is number of * peers, 1 extra for self peer on target * this needs to be tied, host and target * can get out of sync */ num_units = ar->max_num_peers + 1; } else if (num_unit_info & NUM_UNITS_IS_NUM_VDEVS) { num_units = ar->max_num_vdevs + 1; } ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi mem_req_id %d num_units %d num_unit_info %d unit size %d actual units %d\n", req_id, __le32_to_cpu(arg.mem_reqs[i]->num_units), num_unit_info, unit_size, num_units); ret = ath10k_wmi_alloc_host_mem(ar, req_id, num_units, unit_size); if (ret) return; } skip_mem_alloc: ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event service ready min_tx_power 0x%08x max_tx_power 0x%08x ht_cap 0x%08x vht_cap 0x%08x vht_supp_mcs 0x%08x sw_ver0 0x%08x sw_ver1 0x%08x fw_build 0x%08x phy_capab 0x%08x num_rf_chains 0x%08x eeprom_rd 0x%08x low_2ghz_chan %d high_2ghz_chan %d low_5ghz_chan %d high_5ghz_chan %d num_mem_reqs 0x%08x\n", __le32_to_cpu(arg.min_tx_power), __le32_to_cpu(arg.max_tx_power), __le32_to_cpu(arg.ht_cap), __le32_to_cpu(arg.vht_cap), __le32_to_cpu(arg.vht_supp_mcs), __le32_to_cpu(arg.sw_ver0), __le32_to_cpu(arg.sw_ver1), __le32_to_cpu(arg.fw_build), __le32_to_cpu(arg.phy_capab), __le32_to_cpu(arg.num_rf_chains), __le32_to_cpu(arg.eeprom_rd), __le32_to_cpu(arg.low_2ghz_chan), __le32_to_cpu(arg.high_2ghz_chan), __le32_to_cpu(arg.low_5ghz_chan), __le32_to_cpu(arg.high_5ghz_chan), __le32_to_cpu(arg.num_mem_reqs)); dev_kfree_skb(skb); ar->svc_rdy_skb = NULL; complete(&ar->wmi.service_ready); } void ath10k_wmi_event_service_ready(struct ath10k *ar, struct sk_buff *skb) { ar->svc_rdy_skb = skb; queue_work(ar->workqueue_aux, &ar->svc_rdy_work); } static int ath10k_wmi_op_pull_rdy_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_rdy_ev_arg *arg) { struct wmi_ready_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->sw_version = ev->sw_version; arg->abi_version = ev->abi_version; arg->status = ev->status; arg->mac_addr = ev->mac_addr.addr; return 0; } static int ath10k_wmi_op_pull_roam_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_roam_ev_arg *arg) { struct wmi_roam_ev *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->vdev_id = ev->vdev_id; arg->reason = ev->reason; return 0; } static int ath10k_wmi_op_pull_echo_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_echo_ev_arg *arg) { struct wmi_echo_event *ev = (void *)skb->data; arg->value = ev->value; return 0; } int ath10k_wmi_event_ready(struct ath10k *ar, struct sk_buff *skb) { struct wmi_rdy_ev_arg arg = {}; int ret; ret = ath10k_wmi_pull_rdy(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse ready event: %d\n", ret); return ret; } ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event ready sw_version 0x%08x abi_version %u mac_addr %pM status %d\n", __le32_to_cpu(arg.sw_version), __le32_to_cpu(arg.abi_version), arg.mac_addr, __le32_to_cpu(arg.status)); if (is_zero_ether_addr(ar->mac_addr)) ether_addr_copy(ar->mac_addr, arg.mac_addr); complete(&ar->wmi.unified_ready); return 0; } void ath10k_wmi_event_service_available(struct ath10k *ar, struct sk_buff *skb) { int ret; struct wmi_svc_avail_ev_arg arg = {}; ret = ath10k_wmi_pull_svc_avail(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse service available event: %d\n", ret); } /* * Initialization of "arg.service_map_ext_valid" to ZERO is necessary * for the below logic to work. */ if (arg.service_map_ext_valid) ath10k_wmi_map_svc_ext(ar, arg.service_map_ext, ar->wmi.svc_map, __le32_to_cpu(arg.service_map_ext_len)); } static int ath10k_wmi_event_temperature(struct ath10k *ar, struct sk_buff *skb) { const struct wmi_pdev_temperature_event *ev; ev = (struct wmi_pdev_temperature_event *)skb->data; if (WARN_ON(skb->len < sizeof(*ev))) return -EPROTO; ath10k_thermal_event_temperature(ar, __le32_to_cpu(ev->temperature)); return 0; } static int ath10k_wmi_event_pdev_bss_chan_info(struct ath10k *ar, struct sk_buff *skb) { struct wmi_pdev_bss_chan_info_event *ev; struct survey_info *survey; u64 busy, total, tx, rx, rx_bss; u32 freq, noise_floor; u32 cc_freq_hz = ar->hw_params.channel_counters_freq_hz; int idx; ev = (struct wmi_pdev_bss_chan_info_event *)skb->data; if (WARN_ON(skb->len < sizeof(*ev))) return -EPROTO; freq = __le32_to_cpu(ev->freq); noise_floor = __le32_to_cpu(ev->noise_floor); busy = __le64_to_cpu(ev->cycle_busy); total = __le64_to_cpu(ev->cycle_total); tx = __le64_to_cpu(ev->cycle_tx); rx = __le64_to_cpu(ev->cycle_rx); rx_bss = __le64_to_cpu(ev->cycle_rx_bss); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event pdev bss chan info:\n freq: %d noise: %d cycle: busy %llu total %llu tx %llu rx %llu rx_bss %llu\n", freq, noise_floor, busy, total, tx, rx, rx_bss); spin_lock_bh(&ar->data_lock); idx = freq_to_idx(ar, freq); if (idx >= ARRAY_SIZE(ar->survey)) { ath10k_warn(ar, "bss chan info: invalid frequency %d (idx %d out of bounds)\n", freq, idx); goto exit; } survey = &ar->survey[idx]; survey->noise = noise_floor; survey->time = div_u64(total, cc_freq_hz); survey->time_busy = div_u64(busy, cc_freq_hz); survey->time_rx = div_u64(rx_bss, cc_freq_hz); survey->time_tx = div_u64(tx, cc_freq_hz); survey->filled |= (SURVEY_INFO_NOISE_DBM | SURVEY_INFO_TIME | SURVEY_INFO_TIME_BUSY | SURVEY_INFO_TIME_RX | SURVEY_INFO_TIME_TX); exit: spin_unlock_bh(&ar->data_lock); complete(&ar->bss_survey_done); return 0; } static inline void ath10k_wmi_queue_set_coverage_class_work(struct ath10k *ar) { if (ar->hw_params.hw_ops->set_coverage_class) { spin_lock_bh(&ar->data_lock); /* This call only ensures that the modified coverage class * persists in case the firmware sets the registers back to * their default value. So calling it is only necessary if the * coverage class has a non-zero value. */ if (ar->fw_coverage.coverage_class) queue_work(ar->workqueue, &ar->set_coverage_class_work); spin_unlock_bh(&ar->data_lock); } } static void ath10k_wmi_op_rx(struct ath10k *ar, struct sk_buff *skb) { struct wmi_cmd_hdr *cmd_hdr; enum wmi_event_id id; cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = MS(__le32_to_cpu(cmd_hdr->cmd_id), WMI_CMD_HDR_CMD_ID); if (skb_pull(skb, sizeof(struct wmi_cmd_hdr)) == NULL) goto out; trace_ath10k_wmi_event(ar, id, skb->data, skb->len); switch (id) { case WMI_MGMT_RX_EVENTID: ath10k_wmi_event_mgmt_rx(ar, skb); /* mgmt_rx() owns the skb now! */ return; case WMI_SCAN_EVENTID: ath10k_wmi_event_scan(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_CHAN_INFO_EVENTID: ath10k_wmi_event_chan_info(ar, skb); break; case WMI_ECHO_EVENTID: ath10k_wmi_event_echo(ar, skb); break; case WMI_DEBUG_MESG_EVENTID: ath10k_wmi_event_debug_mesg(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_UPDATE_STATS_EVENTID: ath10k_wmi_event_update_stats(ar, skb); break; case WMI_VDEV_START_RESP_EVENTID: ath10k_wmi_event_vdev_start_resp(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_VDEV_STOPPED_EVENTID: ath10k_wmi_event_vdev_stopped(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_PEER_STA_KICKOUT_EVENTID: ath10k_wmi_event_peer_sta_kickout(ar, skb); break; case WMI_HOST_SWBA_EVENTID: ath10k_wmi_event_host_swba(ar, skb); break; case WMI_TBTTOFFSET_UPDATE_EVENTID: ath10k_wmi_event_tbttoffset_update(ar, skb); break; case WMI_PHYERR_EVENTID: ath10k_wmi_event_phyerr(ar, skb); break; case WMI_ROAM_EVENTID: ath10k_wmi_event_roam(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_PROFILE_MATCH: ath10k_wmi_event_profile_match(ar, skb); break; case WMI_DEBUG_PRINT_EVENTID: ath10k_wmi_event_debug_print(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_PDEV_QVIT_EVENTID: ath10k_wmi_event_pdev_qvit(ar, skb); break; case WMI_WLAN_PROFILE_DATA_EVENTID: ath10k_wmi_event_wlan_profile_data(ar, skb); break; case WMI_RTT_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_rtt_measurement_report(ar, skb); break; case WMI_TSF_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_tsf_measurement_report(ar, skb); break; case WMI_RTT_ERROR_REPORT_EVENTID: ath10k_wmi_event_rtt_error_report(ar, skb); break; case WMI_WOW_WAKEUP_HOST_EVENTID: ath10k_wmi_event_wow_wakeup_host(ar, skb); break; case WMI_DCS_INTERFERENCE_EVENTID: ath10k_wmi_event_dcs_interference(ar, skb); break; case WMI_PDEV_TPC_CONFIG_EVENTID: ath10k_wmi_event_pdev_tpc_config(ar, skb); break; case WMI_PDEV_FTM_INTG_EVENTID: ath10k_wmi_event_pdev_ftm_intg(ar, skb); break; case WMI_GTK_OFFLOAD_STATUS_EVENTID: ath10k_wmi_event_gtk_offload_status(ar, skb); break; case WMI_GTK_REKEY_FAIL_EVENTID: ath10k_wmi_event_gtk_rekey_fail(ar, skb); break; case WMI_TX_DELBA_COMPLETE_EVENTID: ath10k_wmi_event_delba_complete(ar, skb); break; case WMI_TX_ADDBA_COMPLETE_EVENTID: ath10k_wmi_event_addba_complete(ar, skb); break; case WMI_VDEV_INSTALL_KEY_COMPLETE_EVENTID: ath10k_wmi_event_vdev_install_key_complete(ar, skb); break; case WMI_SERVICE_READY_EVENTID: ath10k_wmi_event_service_ready(ar, skb); return; case WMI_READY_EVENTID: ath10k_wmi_event_ready(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_SERVICE_AVAILABLE_EVENTID: ath10k_wmi_event_service_available(ar, skb); break; default: ath10k_warn(ar, "Unknown eventid: %d\n", id); break; } out: dev_kfree_skb(skb); } static void ath10k_wmi_10_1_op_rx(struct ath10k *ar, struct sk_buff *skb) { struct wmi_cmd_hdr *cmd_hdr; enum wmi_10x_event_id id; bool consumed; cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = MS(__le32_to_cpu(cmd_hdr->cmd_id), WMI_CMD_HDR_CMD_ID); if (skb_pull(skb, sizeof(struct wmi_cmd_hdr)) == NULL) goto out; trace_ath10k_wmi_event(ar, id, skb->data, skb->len); consumed = ath10k_tm_event_wmi(ar, id, skb); /* Ready event must be handled normally also in UTF mode so that we * know the UTF firmware has booted, others we are just bypass WMI * events to testmode. */ if (consumed && id != WMI_10X_READY_EVENTID) { ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi testmode consumed 0x%x\n", id); goto out; } switch (id) { case WMI_10X_MGMT_RX_EVENTID: ath10k_wmi_event_mgmt_rx(ar, skb); /* mgmt_rx() owns the skb now! */ return; case WMI_10X_SCAN_EVENTID: ath10k_wmi_event_scan(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_CHAN_INFO_EVENTID: ath10k_wmi_event_chan_info(ar, skb); break; case WMI_10X_ECHO_EVENTID: ath10k_wmi_event_echo(ar, skb); break; case WMI_10X_DEBUG_MESG_EVENTID: ath10k_wmi_event_debug_mesg(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_UPDATE_STATS_EVENTID: ath10k_wmi_event_update_stats(ar, skb); break; case WMI_10X_VDEV_START_RESP_EVENTID: ath10k_wmi_event_vdev_start_resp(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_VDEV_STOPPED_EVENTID: ath10k_wmi_event_vdev_stopped(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_PEER_STA_KICKOUT_EVENTID: ath10k_wmi_event_peer_sta_kickout(ar, skb); break; case WMI_10X_HOST_SWBA_EVENTID: ath10k_wmi_event_host_swba(ar, skb); break; case WMI_10X_TBTTOFFSET_UPDATE_EVENTID: ath10k_wmi_event_tbttoffset_update(ar, skb); break; case WMI_10X_PHYERR_EVENTID: ath10k_wmi_event_phyerr(ar, skb); break; case WMI_10X_ROAM_EVENTID: ath10k_wmi_event_roam(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_PROFILE_MATCH: ath10k_wmi_event_profile_match(ar, skb); break; case WMI_10X_DEBUG_PRINT_EVENTID: ath10k_wmi_event_debug_print(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_PDEV_QVIT_EVENTID: ath10k_wmi_event_pdev_qvit(ar, skb); break; case WMI_10X_WLAN_PROFILE_DATA_EVENTID: ath10k_wmi_event_wlan_profile_data(ar, skb); break; case WMI_10X_RTT_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_rtt_measurement_report(ar, skb); break; case WMI_10X_TSF_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_tsf_measurement_report(ar, skb); break; case WMI_10X_RTT_ERROR_REPORT_EVENTID: ath10k_wmi_event_rtt_error_report(ar, skb); break; case WMI_10X_WOW_WAKEUP_HOST_EVENTID: ath10k_wmi_event_wow_wakeup_host(ar, skb); break; case WMI_10X_DCS_INTERFERENCE_EVENTID: ath10k_wmi_event_dcs_interference(ar, skb); break; case WMI_10X_PDEV_TPC_CONFIG_EVENTID: ath10k_wmi_event_pdev_tpc_config(ar, skb); break; case WMI_10X_INST_RSSI_STATS_EVENTID: ath10k_wmi_event_inst_rssi_stats(ar, skb); break; case WMI_10X_VDEV_STANDBY_REQ_EVENTID: ath10k_wmi_event_vdev_standby_req(ar, skb); break; case WMI_10X_VDEV_RESUME_REQ_EVENTID: ath10k_wmi_event_vdev_resume_req(ar, skb); break; case WMI_10X_SERVICE_READY_EVENTID: ath10k_wmi_event_service_ready(ar, skb); return; case WMI_10X_READY_EVENTID: ath10k_wmi_event_ready(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_PDEV_UTF_EVENTID: /* ignore utf events */ break; default: ath10k_warn(ar, "Unknown eventid: %d\n", id); break; } out: dev_kfree_skb(skb); } static void ath10k_wmi_10_2_op_rx(struct ath10k *ar, struct sk_buff *skb) { struct wmi_cmd_hdr *cmd_hdr; enum wmi_10_2_event_id id; bool consumed; cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = MS(__le32_to_cpu(cmd_hdr->cmd_id), WMI_CMD_HDR_CMD_ID); if (skb_pull(skb, sizeof(struct wmi_cmd_hdr)) == NULL) goto out; trace_ath10k_wmi_event(ar, id, skb->data, skb->len); consumed = ath10k_tm_event_wmi(ar, id, skb); /* Ready event must be handled normally also in UTF mode so that we * know the UTF firmware has booted, others we are just bypass WMI * events to testmode. */ if (consumed && id != WMI_10_2_READY_EVENTID) { ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi testmode consumed 0x%x\n", id); goto out; } switch (id) { case WMI_10_2_MGMT_RX_EVENTID: ath10k_wmi_event_mgmt_rx(ar, skb); /* mgmt_rx() owns the skb now! */ return; case WMI_10_2_SCAN_EVENTID: ath10k_wmi_event_scan(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_CHAN_INFO_EVENTID: ath10k_wmi_event_chan_info(ar, skb); break; case WMI_10_2_ECHO_EVENTID: ath10k_wmi_event_echo(ar, skb); break; case WMI_10_2_DEBUG_MESG_EVENTID: ath10k_wmi_event_debug_mesg(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_UPDATE_STATS_EVENTID: ath10k_wmi_event_update_stats(ar, skb); break; case WMI_10_2_VDEV_START_RESP_EVENTID: ath10k_wmi_event_vdev_start_resp(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_VDEV_STOPPED_EVENTID: ath10k_wmi_event_vdev_stopped(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_PEER_STA_KICKOUT_EVENTID: ath10k_wmi_event_peer_sta_kickout(ar, skb); break; case WMI_10_2_HOST_SWBA_EVENTID: ath10k_wmi_event_host_swba(ar, skb); break; case WMI_10_2_TBTTOFFSET_UPDATE_EVENTID: ath10k_wmi_event_tbttoffset_update(ar, skb); break; case WMI_10_2_PHYERR_EVENTID: ath10k_wmi_event_phyerr(ar, skb); break; case WMI_10_2_ROAM_EVENTID: ath10k_wmi_event_roam(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_PROFILE_MATCH: ath10k_wmi_event_profile_match(ar, skb); break; case WMI_10_2_DEBUG_PRINT_EVENTID: ath10k_wmi_event_debug_print(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_PDEV_QVIT_EVENTID: ath10k_wmi_event_pdev_qvit(ar, skb); break; case WMI_10_2_WLAN_PROFILE_DATA_EVENTID: ath10k_wmi_event_wlan_profile_data(ar, skb); break; case WMI_10_2_RTT_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_rtt_measurement_report(ar, skb); break; case WMI_10_2_TSF_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_tsf_measurement_report(ar, skb); break; case WMI_10_2_RTT_ERROR_REPORT_EVENTID: ath10k_wmi_event_rtt_error_report(ar, skb); break; case WMI_10_2_WOW_WAKEUP_HOST_EVENTID: ath10k_wmi_event_wow_wakeup_host(ar, skb); break; case WMI_10_2_DCS_INTERFERENCE_EVENTID: ath10k_wmi_event_dcs_interference(ar, skb); break; case WMI_10_2_PDEV_TPC_CONFIG_EVENTID: ath10k_wmi_event_pdev_tpc_config(ar, skb); break; case WMI_10_2_INST_RSSI_STATS_EVENTID: ath10k_wmi_event_inst_rssi_stats(ar, skb); break; case WMI_10_2_VDEV_STANDBY_REQ_EVENTID: ath10k_wmi_event_vdev_standby_req(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_VDEV_RESUME_REQ_EVENTID: ath10k_wmi_event_vdev_resume_req(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_SERVICE_READY_EVENTID: ath10k_wmi_event_service_ready(ar, skb); return; case WMI_10_2_READY_EVENTID: ath10k_wmi_event_ready(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_PDEV_TEMPERATURE_EVENTID: ath10k_wmi_event_temperature(ar, skb); break; case WMI_10_2_PDEV_BSS_CHAN_INFO_EVENTID: ath10k_wmi_event_pdev_bss_chan_info(ar, skb); break; case WMI_10_2_RTT_KEEPALIVE_EVENTID: case WMI_10_2_GPIO_INPUT_EVENTID: case WMI_10_2_PEER_RATECODE_LIST_EVENTID: case WMI_10_2_GENERIC_BUFFER_EVENTID: case WMI_10_2_MCAST_BUF_RELEASE_EVENTID: case WMI_10_2_MCAST_LIST_AGEOUT_EVENTID: case WMI_10_2_WDS_PEER_EVENTID: ath10k_dbg(ar, ATH10K_DBG_WMI, "received event id %d not implemented\n", id); break; case WMI_10_2_PEER_STA_PS_STATECHG_EVENTID: ath10k_wmi_event_peer_sta_ps_state_chg(ar, skb); break; default: ath10k_warn(ar, "Unknown eventid: %d\n", id); break; } out: dev_kfree_skb(skb); } static void ath10k_wmi_10_4_op_rx(struct ath10k *ar, struct sk_buff *skb) { struct wmi_cmd_hdr *cmd_hdr; enum wmi_10_4_event_id id; bool consumed; cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = MS(__le32_to_cpu(cmd_hdr->cmd_id), WMI_CMD_HDR_CMD_ID); if (!skb_pull(skb, sizeof(struct wmi_cmd_hdr))) goto out; trace_ath10k_wmi_event(ar, id, skb->data, skb->len); consumed = ath10k_tm_event_wmi(ar, id, skb); /* Ready event must be handled normally also in UTF mode so that we * know the UTF firmware has booted, others we are just bypass WMI * events to testmode. */ if (consumed && id != WMI_10_4_READY_EVENTID) { ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi testmode consumed 0x%x\n", id); goto out; } switch (id) { case WMI_10_4_MGMT_RX_EVENTID: ath10k_wmi_event_mgmt_rx(ar, skb); /* mgmt_rx() owns the skb now! */ return; case WMI_10_4_ECHO_EVENTID: ath10k_wmi_event_echo(ar, skb); break; case WMI_10_4_DEBUG_MESG_EVENTID: ath10k_wmi_event_debug_mesg(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_SERVICE_READY_EVENTID: ath10k_wmi_event_service_ready(ar, skb); return; case WMI_10_4_SCAN_EVENTID: ath10k_wmi_event_scan(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_CHAN_INFO_EVENTID: ath10k_wmi_event_chan_info(ar, skb); break; case WMI_10_4_PHYERR_EVENTID: ath10k_wmi_event_phyerr(ar, skb); break; case WMI_10_4_READY_EVENTID: ath10k_wmi_event_ready(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_PEER_STA_KICKOUT_EVENTID: ath10k_wmi_event_peer_sta_kickout(ar, skb); break; case WMI_10_4_ROAM_EVENTID: ath10k_wmi_event_roam(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_HOST_SWBA_EVENTID: ath10k_wmi_event_host_swba(ar, skb); break; case WMI_10_4_TBTTOFFSET_UPDATE_EVENTID: ath10k_wmi_event_tbttoffset_update(ar, skb); break; case WMI_10_4_DEBUG_PRINT_EVENTID: ath10k_wmi_event_debug_print(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_VDEV_START_RESP_EVENTID: ath10k_wmi_event_vdev_start_resp(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_VDEV_STOPPED_EVENTID: ath10k_wmi_event_vdev_stopped(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_WOW_WAKEUP_HOST_EVENTID: case WMI_10_4_PEER_RATECODE_LIST_EVENTID: case WMI_10_4_WDS_PEER_EVENTID: case WMI_10_4_DEBUG_FATAL_CONDITION_EVENTID: ath10k_dbg(ar, ATH10K_DBG_WMI, "received event id %d not implemented\n", id); break; case WMI_10_4_UPDATE_STATS_EVENTID: ath10k_wmi_event_update_stats(ar, skb); break; case WMI_10_4_PDEV_TEMPERATURE_EVENTID: ath10k_wmi_event_temperature(ar, skb); break; case WMI_10_4_PDEV_BSS_CHAN_INFO_EVENTID: ath10k_wmi_event_pdev_bss_chan_info(ar, skb); break; case WMI_10_4_PDEV_TPC_CONFIG_EVENTID: ath10k_wmi_event_pdev_tpc_config(ar, skb); break; case WMI_10_4_TDLS_PEER_EVENTID: ath10k_wmi_handle_tdls_peer_event(ar, skb); break; case WMI_10_4_PDEV_TPC_TABLE_EVENTID: ath10k_wmi_event_tpc_final_table(ar, skb); break; case WMI_10_4_DFS_STATUS_CHECK_EVENTID: ath10k_wmi_event_dfs_status_check(ar, skb); break; case WMI_10_4_PEER_STA_PS_STATECHG_EVENTID: ath10k_wmi_event_peer_sta_ps_state_chg(ar, skb); break; default: ath10k_warn(ar, "Unknown eventid: %d\n", id); break; } out: dev_kfree_skb(skb); } static void ath10k_wmi_process_rx(struct ath10k *ar, struct sk_buff *skb) { int ret; ret = ath10k_wmi_rx(ar, skb); if (ret) ath10k_warn(ar, "failed to process wmi rx: %d\n", ret); } int ath10k_wmi_connect(struct ath10k *ar) { int status; struct ath10k_htc_svc_conn_req conn_req; struct ath10k_htc_svc_conn_resp conn_resp; memset(&ar->wmi.svc_map, 0, sizeof(ar->wmi.svc_map)); memset(&conn_req, 0, sizeof(conn_req)); memset(&conn_resp, 0, sizeof(conn_resp)); /* these fields are the same for all service endpoints */ conn_req.ep_ops.ep_tx_complete = ath10k_wmi_htc_tx_complete; conn_req.ep_ops.ep_rx_complete = ath10k_wmi_process_rx; conn_req.ep_ops.ep_tx_credits = ath10k_wmi_op_ep_tx_credits; /* connect to control service */ conn_req.service_id = ATH10K_HTC_SVC_ID_WMI_CONTROL; status = ath10k_htc_connect_service(&ar->htc, &conn_req, &conn_resp); if (status) { ath10k_warn(ar, "failed to connect to WMI CONTROL service status: %d\n", status); return status; } ar->wmi.eid = conn_resp.eid; return 0; } static struct sk_buff * ath10k_wmi_op_gen_pdev_set_base_macaddr(struct ath10k *ar, const u8 macaddr[ETH_ALEN]) { struct wmi_pdev_set_base_macaddr_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_base_macaddr_cmd *)skb->data; ether_addr_copy(cmd->mac_addr.addr, macaddr); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev basemac %pM\n", macaddr); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pdev_set_rd(struct ath10k *ar, u16 rd, u16 rd2g, u16 rd5g, u16 ctl2g, u16 ctl5g, enum wmi_dfs_region dfs_reg) { struct wmi_pdev_set_regdomain_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_regdomain_cmd *)skb->data; cmd->reg_domain = __cpu_to_le32(rd); cmd->reg_domain_2G = __cpu_to_le32(rd2g); cmd->reg_domain_5G = __cpu_to_le32(rd5g); cmd->conformance_test_limit_2G = __cpu_to_le32(ctl2g); cmd->conformance_test_limit_5G = __cpu_to_le32(ctl5g); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev regdomain rd %x rd2g %x rd5g %x ctl2g %x ctl5g %x\n", rd, rd2g, rd5g, ctl2g, ctl5g); return skb; } static struct sk_buff * ath10k_wmi_10x_op_gen_pdev_set_rd(struct ath10k *ar, u16 rd, u16 rd2g, u16 rd5g, u16 ctl2g, u16 ctl5g, enum wmi_dfs_region dfs_reg) { struct wmi_pdev_set_regdomain_cmd_10x *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_regdomain_cmd_10x *)skb->data; cmd->reg_domain = __cpu_to_le32(rd); cmd->reg_domain_2G = __cpu_to_le32(rd2g); cmd->reg_domain_5G = __cpu_to_le32(rd5g); cmd->conformance_test_limit_2G = __cpu_to_le32(ctl2g); cmd->conformance_test_limit_5G = __cpu_to_le32(ctl5g); cmd->dfs_domain = __cpu_to_le32(dfs_reg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev regdomain rd %x rd2g %x rd5g %x ctl2g %x ctl5g %x dfs_region %x\n", rd, rd2g, rd5g, ctl2g, ctl5g, dfs_reg); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pdev_suspend(struct ath10k *ar, u32 suspend_opt) { struct wmi_pdev_suspend_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_suspend_cmd *)skb->data; cmd->suspend_opt = __cpu_to_le32(suspend_opt); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pdev_resume(struct ath10k *ar) { struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, 0); if (!skb) return ERR_PTR(-ENOMEM); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pdev_set_param(struct ath10k *ar, u32 id, u32 value) { struct wmi_pdev_set_param_cmd *cmd; struct sk_buff *skb; if (id == WMI_PDEV_PARAM_UNSUPPORTED) { ath10k_warn(ar, "pdev param %d not supported by firmware\n", id); return ERR_PTR(-EOPNOTSUPP); } skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_param_cmd *)skb->data; cmd->param_id = __cpu_to_le32(id); cmd->param_value = __cpu_to_le32(value); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev set param %d value %d\n", id, value); return skb; } void ath10k_wmi_put_host_mem_chunks(struct ath10k *ar, struct wmi_host_mem_chunks *chunks) { struct host_memory_chunk *chunk; int i; chunks->count = __cpu_to_le32(ar->wmi.num_mem_chunks); for (i = 0; i < ar->wmi.num_mem_chunks; i++) { chunk = &chunks->items[i]; chunk->ptr = __cpu_to_le32(ar->wmi.mem_chunks[i].paddr); chunk->size = __cpu_to_le32(ar->wmi.mem_chunks[i].len); chunk->req_id = __cpu_to_le32(ar->wmi.mem_chunks[i].req_id); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi chunk %d len %d requested, addr 0x%llx\n", i, ar->wmi.mem_chunks[i].len, (unsigned long long)ar->wmi.mem_chunks[i].paddr); } } static struct sk_buff *ath10k_wmi_op_gen_init(struct ath10k *ar) { struct wmi_init_cmd *cmd; struct sk_buff *buf; struct wmi_resource_config config = {}; u32 val; config.num_vdevs = __cpu_to_le32(TARGET_NUM_VDEVS); config.num_peers = __cpu_to_le32(TARGET_NUM_PEERS); config.num_offload_peers = __cpu_to_le32(TARGET_NUM_OFFLOAD_PEERS); config.num_offload_reorder_bufs = __cpu_to_le32(TARGET_NUM_OFFLOAD_REORDER_BUFS); config.num_peer_keys = __cpu_to_le32(TARGET_NUM_PEER_KEYS); config.num_tids = __cpu_to_le32(TARGET_NUM_TIDS); config.ast_skid_limit = __cpu_to_le32(TARGET_AST_SKID_LIMIT); config.tx_chain_mask = __cpu_to_le32(TARGET_TX_CHAIN_MASK); config.rx_chain_mask = __cpu_to_le32(TARGET_RX_CHAIN_MASK); config.rx_timeout_pri_vo = __cpu_to_le32(TARGET_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_vi = __cpu_to_le32(TARGET_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_be = __cpu_to_le32(TARGET_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_bk = __cpu_to_le32(TARGET_RX_TIMEOUT_HI_PRI); config.rx_decap_mode = __cpu_to_le32(ar->wmi.rx_decap_mode); config.scan_max_pending_reqs = __cpu_to_le32(TARGET_SCAN_MAX_PENDING_REQS); config.bmiss_offload_max_vdev = __cpu_to_le32(TARGET_BMISS_OFFLOAD_MAX_VDEV); config.roam_offload_max_vdev = __cpu_to_le32(TARGET_ROAM_OFFLOAD_MAX_VDEV); config.roam_offload_max_ap_profiles = __cpu_to_le32(TARGET_ROAM_OFFLOAD_MAX_AP_PROFILES); config.num_mcast_groups = __cpu_to_le32(TARGET_NUM_MCAST_GROUPS); config.num_mcast_table_elems = __cpu_to_le32(TARGET_NUM_MCAST_TABLE_ELEMS); config.mcast2ucast_mode = __cpu_to_le32(TARGET_MCAST2UCAST_MODE); config.tx_dbg_log_size = __cpu_to_le32(TARGET_TX_DBG_LOG_SIZE); config.num_wds_entries = __cpu_to_le32(TARGET_NUM_WDS_ENTRIES); config.dma_burst_size = __cpu_to_le32(TARGET_DMA_BURST_SIZE); config.mac_aggr_delim = __cpu_to_le32(TARGET_MAC_AGGR_DELIM); val = TARGET_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK; config.rx_skip_defrag_timeout_dup_detection_check = __cpu_to_le32(val); config.vow_config = __cpu_to_le32(TARGET_VOW_CONFIG); config.gtk_offload_max_vdev = __cpu_to_le32(TARGET_GTK_OFFLOAD_MAX_VDEV); config.num_msdu_desc = __cpu_to_le32(TARGET_NUM_MSDU_DESC); config.max_frag_entries = __cpu_to_le32(TARGET_MAX_FRAG_ENTRIES); buf = ath10k_wmi_alloc_skb(ar, struct_size(cmd, mem_chunks.items, ar->wmi.num_mem_chunks)); if (!buf) return ERR_PTR(-ENOMEM); cmd = (struct wmi_init_cmd *)buf->data; memcpy(&cmd->resource_config, &config, sizeof(config)); ath10k_wmi_put_host_mem_chunks(ar, &cmd->mem_chunks); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi init\n"); return buf; } static struct sk_buff *ath10k_wmi_10_1_op_gen_init(struct ath10k *ar) { struct wmi_init_cmd_10x *cmd; struct sk_buff *buf; struct wmi_resource_config_10x config = {}; u32 val; config.num_vdevs = __cpu_to_le32(TARGET_10X_NUM_VDEVS); config.num_peers = __cpu_to_le32(TARGET_10X_NUM_PEERS); config.num_peer_keys = __cpu_to_le32(TARGET_10X_NUM_PEER_KEYS); config.num_tids = __cpu_to_le32(TARGET_10X_NUM_TIDS); config.ast_skid_limit = __cpu_to_le32(TARGET_10X_AST_SKID_LIMIT); config.tx_chain_mask = __cpu_to_le32(TARGET_10X_TX_CHAIN_MASK); config.rx_chain_mask = __cpu_to_le32(TARGET_10X_RX_CHAIN_MASK); config.rx_timeout_pri_vo = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_vi = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_be = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_bk = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_HI_PRI); config.rx_decap_mode = __cpu_to_le32(ar->wmi.rx_decap_mode); config.scan_max_pending_reqs = __cpu_to_le32(TARGET_10X_SCAN_MAX_PENDING_REQS); config.bmiss_offload_max_vdev = __cpu_to_le32(TARGET_10X_BMISS_OFFLOAD_MAX_VDEV); config.roam_offload_max_vdev = __cpu_to_le32(TARGET_10X_ROAM_OFFLOAD_MAX_VDEV); config.roam_offload_max_ap_profiles = __cpu_to_le32(TARGET_10X_ROAM_OFFLOAD_MAX_AP_PROFILES); config.num_mcast_groups = __cpu_to_le32(TARGET_10X_NUM_MCAST_GROUPS); config.num_mcast_table_elems = __cpu_to_le32(TARGET_10X_NUM_MCAST_TABLE_ELEMS); config.mcast2ucast_mode = __cpu_to_le32(TARGET_10X_MCAST2UCAST_MODE); config.tx_dbg_log_size = __cpu_to_le32(TARGET_10X_TX_DBG_LOG_SIZE); config.num_wds_entries = __cpu_to_le32(TARGET_10X_NUM_WDS_ENTRIES); config.dma_burst_size = __cpu_to_le32(TARGET_10X_DMA_BURST_SIZE); config.mac_aggr_delim = __cpu_to_le32(TARGET_10X_MAC_AGGR_DELIM); val = TARGET_10X_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK; config.rx_skip_defrag_timeout_dup_detection_check = __cpu_to_le32(val); config.vow_config = __cpu_to_le32(TARGET_10X_VOW_CONFIG); config.num_msdu_desc = __cpu_to_le32(TARGET_10X_NUM_MSDU_DESC); config.max_frag_entries = __cpu_to_le32(TARGET_10X_MAX_FRAG_ENTRIES); buf = ath10k_wmi_alloc_skb(ar, struct_size(cmd, mem_chunks.items, ar->wmi.num_mem_chunks)); if (!buf) return ERR_PTR(-ENOMEM); cmd = (struct wmi_init_cmd_10x *)buf->data; memcpy(&cmd->resource_config, &config, sizeof(config)); ath10k_wmi_put_host_mem_chunks(ar, &cmd->mem_chunks); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi init 10x\n"); return buf; } static struct sk_buff *ath10k_wmi_10_2_op_gen_init(struct ath10k *ar) { struct wmi_init_cmd_10_2 *cmd; struct sk_buff *buf; struct wmi_resource_config_10x config = {}; u32 val, features; config.num_vdevs = __cpu_to_le32(TARGET_10X_NUM_VDEVS); config.num_peer_keys = __cpu_to_le32(TARGET_10X_NUM_PEER_KEYS); if (ath10k_peer_stats_enabled(ar)) { config.num_peers = __cpu_to_le32(TARGET_10X_TX_STATS_NUM_PEERS); config.num_tids = __cpu_to_le32(TARGET_10X_TX_STATS_NUM_TIDS); } else { config.num_peers = __cpu_to_le32(TARGET_10X_NUM_PEERS); config.num_tids = __cpu_to_le32(TARGET_10X_NUM_TIDS); } config.ast_skid_limit = __cpu_to_le32(TARGET_10X_AST_SKID_LIMIT); config.tx_chain_mask = __cpu_to_le32(TARGET_10X_TX_CHAIN_MASK); config.rx_chain_mask = __cpu_to_le32(TARGET_10X_RX_CHAIN_MASK); config.rx_timeout_pri_vo = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_vi = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_be = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_bk = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_HI_PRI); config.rx_decap_mode = __cpu_to_le32(ar->wmi.rx_decap_mode); config.scan_max_pending_reqs = __cpu_to_le32(TARGET_10X_SCAN_MAX_PENDING_REQS); config.bmiss_offload_max_vdev = __cpu_to_le32(TARGET_10X_BMISS_OFFLOAD_MAX_VDEV); config.roam_offload_max_vdev = __cpu_to_le32(TARGET_10X_ROAM_OFFLOAD_MAX_VDEV); config.roam_offload_max_ap_profiles = __cpu_to_le32(TARGET_10X_ROAM_OFFLOAD_MAX_AP_PROFILES); config.num_mcast_groups = __cpu_to_le32(TARGET_10X_NUM_MCAST_GROUPS); config.num_mcast_table_elems = __cpu_to_le32(TARGET_10X_NUM_MCAST_TABLE_ELEMS); config.mcast2ucast_mode = __cpu_to_le32(TARGET_10X_MCAST2UCAST_MODE); config.tx_dbg_log_size = __cpu_to_le32(TARGET_10X_TX_DBG_LOG_SIZE); config.num_wds_entries = __cpu_to_le32(TARGET_10X_NUM_WDS_ENTRIES); config.dma_burst_size = __cpu_to_le32(TARGET_10_2_DMA_BURST_SIZE); config.mac_aggr_delim = __cpu_to_le32(TARGET_10X_MAC_AGGR_DELIM); val = TARGET_10X_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK; config.rx_skip_defrag_timeout_dup_detection_check = __cpu_to_le32(val); config.vow_config = __cpu_to_le32(TARGET_10X_VOW_CONFIG); config.num_msdu_desc = __cpu_to_le32(TARGET_10X_NUM_MSDU_DESC); config.max_frag_entries = __cpu_to_le32(TARGET_10X_MAX_FRAG_ENTRIES); buf = ath10k_wmi_alloc_skb(ar, struct_size(cmd, mem_chunks.items, ar->wmi.num_mem_chunks)); if (!buf) return ERR_PTR(-ENOMEM); cmd = (struct wmi_init_cmd_10_2 *)buf->data; features = WMI_10_2_RX_BATCH_MODE; if (test_bit(ATH10K_FLAG_BTCOEX, &ar->dev_flags) && test_bit(WMI_SERVICE_COEX_GPIO, ar->wmi.svc_map)) features |= WMI_10_2_COEX_GPIO; if (ath10k_peer_stats_enabled(ar)) features |= WMI_10_2_PEER_STATS; if (test_bit(WMI_SERVICE_BSS_CHANNEL_INFO_64, ar->wmi.svc_map)) features |= WMI_10_2_BSS_CHAN_INFO; cmd->resource_config.feature_mask = __cpu_to_le32(features); memcpy(&cmd->resource_config.common, &config, sizeof(config)); ath10k_wmi_put_host_mem_chunks(ar, &cmd->mem_chunks); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi init 10.2\n"); return buf; } static struct sk_buff *ath10k_wmi_10_4_op_gen_init(struct ath10k *ar) { struct wmi_init_cmd_10_4 *cmd; struct sk_buff *buf; struct wmi_resource_config_10_4 config = {}; config.num_vdevs = __cpu_to_le32(ar->max_num_vdevs); config.num_peers = __cpu_to_le32(ar->max_num_peers); config.num_active_peers = __cpu_to_le32(ar->num_active_peers); config.num_tids = __cpu_to_le32(ar->num_tids); config.num_offload_peers = __cpu_to_le32(TARGET_10_4_NUM_OFFLOAD_PEERS); config.num_offload_reorder_buffs = __cpu_to_le32(TARGET_10_4_NUM_OFFLOAD_REORDER_BUFFS); config.num_peer_keys = __cpu_to_le32(TARGET_10_4_NUM_PEER_KEYS); config.ast_skid_limit = __cpu_to_le32(TARGET_10_4_AST_SKID_LIMIT); config.tx_chain_mask = __cpu_to_le32(ar->hw_params.tx_chain_mask); config.rx_chain_mask = __cpu_to_le32(ar->hw_params.rx_chain_mask); config.rx_timeout_pri[0] = __cpu_to_le32(TARGET_10_4_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri[1] = __cpu_to_le32(TARGET_10_4_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri[2] = __cpu_to_le32(TARGET_10_4_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri[3] = __cpu_to_le32(TARGET_10_4_RX_TIMEOUT_HI_PRI); config.rx_decap_mode = __cpu_to_le32(ar->wmi.rx_decap_mode); config.scan_max_pending_req = __cpu_to_le32(TARGET_10_4_SCAN_MAX_REQS); config.bmiss_offload_max_vdev = __cpu_to_le32(TARGET_10_4_BMISS_OFFLOAD_MAX_VDEV); config.roam_offload_max_vdev = __cpu_to_le32(TARGET_10_4_ROAM_OFFLOAD_MAX_VDEV); config.roam_offload_max_ap_profiles = __cpu_to_le32(TARGET_10_4_ROAM_OFFLOAD_MAX_PROFILES); config.num_mcast_groups = __cpu_to_le32(TARGET_10_4_NUM_MCAST_GROUPS); config.num_mcast_table_elems = __cpu_to_le32(TARGET_10_4_NUM_MCAST_TABLE_ELEMS); config.mcast2ucast_mode = __cpu_to_le32(TARGET_10_4_MCAST2UCAST_MODE); config.tx_dbg_log_size = __cpu_to_le32(TARGET_10_4_TX_DBG_LOG_SIZE); config.num_wds_entries = __cpu_to_le32(TARGET_10_4_NUM_WDS_ENTRIES); config.dma_burst_size = __cpu_to_le32(TARGET_10_4_DMA_BURST_SIZE); config.mac_aggr_delim = __cpu_to_le32(TARGET_10_4_MAC_AGGR_DELIM); config.rx_skip_defrag_timeout_dup_detection_check = __cpu_to_le32(TARGET_10_4_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK); config.vow_config = __cpu_to_le32(TARGET_10_4_VOW_CONFIG); config.gtk_offload_max_vdev = __cpu_to_le32(TARGET_10_4_GTK_OFFLOAD_MAX_VDEV); config.num_msdu_desc = __cpu_to_le32(ar->htt.max_num_pending_tx); config.max_frag_entries = __cpu_to_le32(TARGET_10_4_11AC_TX_MAX_FRAGS); config.max_peer_ext_stats = __cpu_to_le32(TARGET_10_4_MAX_PEER_EXT_STATS); config.smart_ant_cap = __cpu_to_le32(TARGET_10_4_SMART_ANT_CAP); config.bk_minfree = __cpu_to_le32(TARGET_10_4_BK_MIN_FREE); config.be_minfree = __cpu_to_le32(TARGET_10_4_BE_MIN_FREE); config.vi_minfree = __cpu_to_le32(TARGET_10_4_VI_MIN_FREE); config.vo_minfree = __cpu_to_le32(TARGET_10_4_VO_MIN_FREE); config.rx_batchmode = __cpu_to_le32(TARGET_10_4_RX_BATCH_MODE); config.tt_support = __cpu_to_le32(TARGET_10_4_THERMAL_THROTTLING_CONFIG); config.atf_config = __cpu_to_le32(TARGET_10_4_ATF_CONFIG); config.iphdr_pad_config = __cpu_to_le32(TARGET_10_4_IPHDR_PAD_CONFIG); config.qwrap_config = __cpu_to_le32(TARGET_10_4_QWRAP_CONFIG); buf = ath10k_wmi_alloc_skb(ar, struct_size(cmd, mem_chunks.items, ar->wmi.num_mem_chunks)); if (!buf) return ERR_PTR(-ENOMEM); cmd = (struct wmi_init_cmd_10_4 *)buf->data; memcpy(&cmd->resource_config, &config, sizeof(config)); ath10k_wmi_put_host_mem_chunks(ar, &cmd->mem_chunks); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi init 10.4\n"); return buf; } int ath10k_wmi_start_scan_verify(const struct wmi_start_scan_arg *arg) { if (arg->ie_len > WLAN_SCAN_PARAMS_MAX_IE_LEN) return -EINVAL; if (arg->n_channels > ARRAY_SIZE(arg->channels)) return -EINVAL; if (arg->n_ssids > WLAN_SCAN_PARAMS_MAX_SSID) return -EINVAL; if (arg->n_bssids > WLAN_SCAN_PARAMS_MAX_BSSID) return -EINVAL; return 0; } static size_t ath10k_wmi_start_scan_tlvs_len(const struct wmi_start_scan_arg *arg) { int len = 0; if (arg->ie_len) { len += sizeof(struct wmi_ie_data); len += roundup(arg->ie_len, 4); } if (arg->n_channels) { len += sizeof(struct wmi_chan_list); len += sizeof(__le32) * arg->n_channels; } if (arg->n_ssids) { len += sizeof(struct wmi_ssid_list); len += sizeof(struct wmi_ssid) * arg->n_ssids; } if (arg->n_bssids) { len += sizeof(struct wmi_bssid_list); len += sizeof(struct wmi_mac_addr) * arg->n_bssids; } return len; } void ath10k_wmi_put_start_scan_common(struct wmi_start_scan_common *cmn, const struct wmi_start_scan_arg *arg) { u32 scan_id; u32 scan_req_id; scan_id = WMI_HOST_SCAN_REQ_ID_PREFIX; scan_id |= arg->scan_id; scan_req_id = WMI_HOST_SCAN_REQUESTOR_ID_PREFIX; scan_req_id |= arg->scan_req_id; cmn->scan_id = __cpu_to_le32(scan_id); cmn->scan_req_id = __cpu_to_le32(scan_req_id); cmn->vdev_id = __cpu_to_le32(arg->vdev_id); cmn->scan_priority = __cpu_to_le32(arg->scan_priority); cmn->notify_scan_events = __cpu_to_le32(arg->notify_scan_events); cmn->dwell_time_active = __cpu_to_le32(arg->dwell_time_active); cmn->dwell_time_passive = __cpu_to_le32(arg->dwell_time_passive); cmn->min_rest_time = __cpu_to_le32(arg->min_rest_time); cmn->max_rest_time = __cpu_to_le32(arg->max_rest_time); cmn->repeat_probe_time = __cpu_to_le32(arg->repeat_probe_time); cmn->probe_spacing_time = __cpu_to_le32(arg->probe_spacing_time); cmn->idle_time = __cpu_to_le32(arg->idle_time); cmn->max_scan_time = __cpu_to_le32(arg->max_scan_time); cmn->probe_delay = __cpu_to_le32(arg->probe_delay); cmn->scan_ctrl_flags = __cpu_to_le32(arg->scan_ctrl_flags); } static void ath10k_wmi_put_start_scan_tlvs(u8 *tlvs, const struct wmi_start_scan_arg *arg) { struct wmi_ie_data *ie; struct wmi_chan_list *channels; struct wmi_ssid_list *ssids; struct wmi_bssid_list *bssids; void *ptr = tlvs; int i; if (arg->n_channels) { channels = ptr; channels->tag = __cpu_to_le32(WMI_CHAN_LIST_TAG); channels->num_chan = __cpu_to_le32(arg->n_channels); for (i = 0; i < arg->n_channels; i++) channels->channel_list[i].freq = __cpu_to_le16(arg->channels[i]); ptr += sizeof(*channels); ptr += sizeof(__le32) * arg->n_channels; } if (arg->n_ssids) { ssids = ptr; ssids->tag = __cpu_to_le32(WMI_SSID_LIST_TAG); ssids->num_ssids = __cpu_to_le32(arg->n_ssids); for (i = 0; i < arg->n_ssids; i++) { ssids->ssids[i].ssid_len = __cpu_to_le32(arg->ssids[i].len); memcpy(&ssids->ssids[i].ssid, arg->ssids[i].ssid, arg->ssids[i].len); } ptr += sizeof(*ssids); ptr += sizeof(struct wmi_ssid) * arg->n_ssids; } if (arg->n_bssids) { bssids = ptr; bssids->tag = __cpu_to_le32(WMI_BSSID_LIST_TAG); bssids->num_bssid = __cpu_to_le32(arg->n_bssids); for (i = 0; i < arg->n_bssids; i++) ether_addr_copy(bssids->bssid_list[i].addr, arg->bssids[i].bssid); ptr += sizeof(*bssids); ptr += sizeof(struct wmi_mac_addr) * arg->n_bssids; } if (arg->ie_len) { ie = ptr; ie->tag = __cpu_to_le32(WMI_IE_TAG); ie->ie_len = __cpu_to_le32(arg->ie_len); memcpy(ie->ie_data, arg->ie, arg->ie_len); ptr += sizeof(*ie); ptr += roundup(arg->ie_len, 4); } } static struct sk_buff * ath10k_wmi_op_gen_start_scan(struct ath10k *ar, const struct wmi_start_scan_arg *arg) { struct wmi_start_scan_cmd *cmd; struct sk_buff *skb; size_t len; int ret; ret = ath10k_wmi_start_scan_verify(arg); if (ret) return ERR_PTR(ret); len = sizeof(*cmd) + ath10k_wmi_start_scan_tlvs_len(arg); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_start_scan_cmd *)skb->data; ath10k_wmi_put_start_scan_common(&cmd->common, arg); ath10k_wmi_put_start_scan_tlvs(cmd->tlvs, arg); cmd->burst_duration_ms = __cpu_to_le32(0); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi start scan\n"); return skb; } static struct sk_buff * ath10k_wmi_10x_op_gen_start_scan(struct ath10k *ar, const struct wmi_start_scan_arg *arg) { struct wmi_10x_start_scan_cmd *cmd; struct sk_buff *skb; size_t len; int ret; ret = ath10k_wmi_start_scan_verify(arg); if (ret) return ERR_PTR(ret); len = sizeof(*cmd) + ath10k_wmi_start_scan_tlvs_len(arg); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_10x_start_scan_cmd *)skb->data; ath10k_wmi_put_start_scan_common(&cmd->common, arg); ath10k_wmi_put_start_scan_tlvs(cmd->tlvs, arg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi 10x start scan\n"); return skb; } void ath10k_wmi_start_scan_init(struct ath10k *ar, struct wmi_start_scan_arg *arg) { /* setup commonly used values */ arg->scan_req_id = 1; arg->scan_priority = WMI_SCAN_PRIORITY_LOW; arg->dwell_time_active = 50; arg->dwell_time_passive = 150; arg->min_rest_time = 50; arg->max_rest_time = 500; arg->repeat_probe_time = 0; arg->probe_spacing_time = 0; arg->idle_time = 0; arg->max_scan_time = 20000; arg->probe_delay = 5; arg->notify_scan_events = WMI_SCAN_EVENT_STARTED | WMI_SCAN_EVENT_COMPLETED | WMI_SCAN_EVENT_BSS_CHANNEL | WMI_SCAN_EVENT_FOREIGN_CHANNEL | WMI_SCAN_EVENT_FOREIGN_CHANNEL_EXIT | WMI_SCAN_EVENT_DEQUEUED; arg->scan_ctrl_flags |= WMI_SCAN_CHAN_STAT_EVENT; arg->n_bssids = 1; arg->bssids[0].bssid = "\xFF\xFF\xFF\xFF\xFF\xFF"; } static struct sk_buff * ath10k_wmi_op_gen_stop_scan(struct ath10k *ar, const struct wmi_stop_scan_arg *arg) { struct wmi_stop_scan_cmd *cmd; struct sk_buff *skb; u32 scan_id; u32 req_id; if (arg->req_id > 0xFFF) return ERR_PTR(-EINVAL); if (arg->req_type == WMI_SCAN_STOP_ONE && arg->u.scan_id > 0xFFF) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); scan_id = arg->u.scan_id; scan_id |= WMI_HOST_SCAN_REQ_ID_PREFIX; req_id = arg->req_id; req_id |= WMI_HOST_SCAN_REQUESTOR_ID_PREFIX; cmd = (struct wmi_stop_scan_cmd *)skb->data; cmd->req_type = __cpu_to_le32(arg->req_type); cmd->vdev_id = __cpu_to_le32(arg->u.vdev_id); cmd->scan_id = __cpu_to_le32(scan_id); cmd->scan_req_id = __cpu_to_le32(req_id); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi stop scan reqid %d req_type %d vdev/scan_id %d\n", arg->req_id, arg->req_type, arg->u.scan_id); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_create(struct ath10k *ar, u32 vdev_id, enum wmi_vdev_type type, enum wmi_vdev_subtype subtype, const u8 macaddr[ETH_ALEN]) { struct wmi_vdev_create_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_create_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->vdev_type = __cpu_to_le32(type); cmd->vdev_subtype = __cpu_to_le32(subtype); ether_addr_copy(cmd->vdev_macaddr.addr, macaddr); ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI vdev create: id %d type %d subtype %d macaddr %pM\n", vdev_id, type, subtype, macaddr); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_delete(struct ath10k *ar, u32 vdev_id) { struct wmi_vdev_delete_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_delete_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI vdev delete id %d\n", vdev_id); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_start(struct ath10k *ar, const struct wmi_vdev_start_request_arg *arg, bool restart) { struct wmi_vdev_start_request_cmd *cmd; struct sk_buff *skb; const char *cmdname; u32 flags = 0; if (WARN_ON(arg->hidden_ssid && !arg->ssid)) return ERR_PTR(-EINVAL); if (WARN_ON(arg->ssid_len > sizeof(cmd->ssid.ssid))) return ERR_PTR(-EINVAL); if (restart) cmdname = "restart"; else cmdname = "start"; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); if (arg->hidden_ssid) flags |= WMI_VDEV_START_HIDDEN_SSID; if (arg->pmf_enabled) flags |= WMI_VDEV_START_PMF_ENABLED; cmd = (struct wmi_vdev_start_request_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(arg->vdev_id); cmd->disable_hw_ack = __cpu_to_le32(arg->disable_hw_ack); cmd->beacon_interval = __cpu_to_le32(arg->bcn_intval); cmd->dtim_period = __cpu_to_le32(arg->dtim_period); cmd->flags = __cpu_to_le32(flags); cmd->bcn_tx_rate = __cpu_to_le32(arg->bcn_tx_rate); cmd->bcn_tx_power = __cpu_to_le32(arg->bcn_tx_power); if (arg->ssid) { cmd->ssid.ssid_len = __cpu_to_le32(arg->ssid_len); memcpy(cmd->ssid.ssid, arg->ssid, arg->ssid_len); } ath10k_wmi_put_wmi_channel(ar, &cmd->chan, &arg->channel); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi vdev %s id 0x%x flags: 0x%0X, freq %d, mode %d, ch_flags: 0x%0X, max_power: %d\n", cmdname, arg->vdev_id, flags, arg->channel.freq, arg->channel.mode, cmd->chan.flags, arg->channel.max_power); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_stop(struct ath10k *ar, u32 vdev_id) { struct wmi_vdev_stop_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_stop_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi vdev stop id 0x%x\n", vdev_id); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_up(struct ath10k *ar, u32 vdev_id, u32 aid, const u8 *bssid) { struct wmi_vdev_up_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_up_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->vdev_assoc_id = __cpu_to_le32(aid); ether_addr_copy(cmd->vdev_bssid.addr, bssid); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi mgmt vdev up id 0x%x assoc id %d bssid %pM\n", vdev_id, aid, bssid); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_down(struct ath10k *ar, u32 vdev_id) { struct wmi_vdev_down_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_down_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi mgmt vdev down id 0x%x\n", vdev_id); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_set_param(struct ath10k *ar, u32 vdev_id, u32 param_id, u32 param_value) { struct wmi_vdev_set_param_cmd *cmd; struct sk_buff *skb; if (param_id == WMI_VDEV_PARAM_UNSUPPORTED) { ath10k_dbg(ar, ATH10K_DBG_WMI, "vdev param %d not supported by firmware\n", param_id); return ERR_PTR(-EOPNOTSUPP); } skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_set_param_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->param_id = __cpu_to_le32(param_id); cmd->param_value = __cpu_to_le32(param_value); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi vdev id 0x%x set param %d value %d\n", vdev_id, param_id, param_value); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_install_key(struct ath10k *ar, const struct wmi_vdev_install_key_arg *arg) { struct wmi_vdev_install_key_cmd *cmd; struct sk_buff *skb; if (arg->key_cipher == WMI_CIPHER_NONE && arg->key_data != NULL) return ERR_PTR(-EINVAL); if (arg->key_cipher != WMI_CIPHER_NONE && arg->key_data == NULL) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd) + arg->key_len); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_install_key_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(arg->vdev_id); cmd->key_idx = __cpu_to_le32(arg->key_idx); cmd->key_flags = __cpu_to_le32(arg->key_flags); cmd->key_cipher = __cpu_to_le32(arg->key_cipher); cmd->key_len = __cpu_to_le32(arg->key_len); cmd->key_txmic_len = __cpu_to_le32(arg->key_txmic_len); cmd->key_rxmic_len = __cpu_to_le32(arg->key_rxmic_len); if (arg->macaddr) ether_addr_copy(cmd->peer_macaddr.addr, arg->macaddr); if (arg->key_data) memcpy(cmd->key_data, arg->key_data, arg->key_len); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi vdev install key idx %d cipher %d len %d\n", arg->key_idx, arg->key_cipher, arg->key_len); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_spectral_conf(struct ath10k *ar, const struct wmi_vdev_spectral_conf_arg *arg) { struct wmi_vdev_spectral_conf_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_spectral_conf_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(arg->vdev_id); cmd->scan_count = __cpu_to_le32(arg->scan_count); cmd->scan_period = __cpu_to_le32(arg->scan_period); cmd->scan_priority = __cpu_to_le32(arg->scan_priority); cmd->scan_fft_size = __cpu_to_le32(arg->scan_fft_size); cmd->scan_gc_ena = __cpu_to_le32(arg->scan_gc_ena); cmd->scan_restart_ena = __cpu_to_le32(arg->scan_restart_ena); cmd->scan_noise_floor_ref = __cpu_to_le32(arg->scan_noise_floor_ref); cmd->scan_init_delay = __cpu_to_le32(arg->scan_init_delay); cmd->scan_nb_tone_thr = __cpu_to_le32(arg->scan_nb_tone_thr); cmd->scan_str_bin_thr = __cpu_to_le32(arg->scan_str_bin_thr); cmd->scan_wb_rpt_mode = __cpu_to_le32(arg->scan_wb_rpt_mode); cmd->scan_rssi_rpt_mode = __cpu_to_le32(arg->scan_rssi_rpt_mode); cmd->scan_rssi_thr = __cpu_to_le32(arg->scan_rssi_thr); cmd->scan_pwr_format = __cpu_to_le32(arg->scan_pwr_format); cmd->scan_rpt_mode = __cpu_to_le32(arg->scan_rpt_mode); cmd->scan_bin_scale = __cpu_to_le32(arg->scan_bin_scale); cmd->scan_dbm_adj = __cpu_to_le32(arg->scan_dbm_adj); cmd->scan_chn_mask = __cpu_to_le32(arg->scan_chn_mask); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_spectral_enable(struct ath10k *ar, u32 vdev_id, u32 trigger, u32 enable) { struct wmi_vdev_spectral_enable_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_spectral_enable_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->trigger_cmd = __cpu_to_le32(trigger); cmd->enable_cmd = __cpu_to_le32(enable); return skb; } static struct sk_buff * ath10k_wmi_op_gen_peer_create(struct ath10k *ar, u32 vdev_id, const u8 peer_addr[ETH_ALEN], enum wmi_peer_type peer_type) { struct wmi_peer_create_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_peer_create_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, peer_addr); cmd->peer_type = __cpu_to_le32(peer_type); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer create vdev_id %d peer_addr %pM\n", vdev_id, peer_addr); return skb; } static struct sk_buff * ath10k_wmi_op_gen_peer_delete(struct ath10k *ar, u32 vdev_id, const u8 peer_addr[ETH_ALEN]) { struct wmi_peer_delete_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_peer_delete_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, peer_addr); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer delete vdev_id %d peer_addr %pM\n", vdev_id, peer_addr); return skb; } static struct sk_buff * ath10k_wmi_op_gen_peer_flush(struct ath10k *ar, u32 vdev_id, const u8 peer_addr[ETH_ALEN], u32 tid_bitmap) { struct wmi_peer_flush_tids_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_peer_flush_tids_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->peer_tid_bitmap = __cpu_to_le32(tid_bitmap); ether_addr_copy(cmd->peer_macaddr.addr, peer_addr); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer flush vdev_id %d peer_addr %pM tids %08x\n", vdev_id, peer_addr, tid_bitmap); return skb; } static struct sk_buff * ath10k_wmi_op_gen_peer_set_param(struct ath10k *ar, u32 vdev_id, const u8 *peer_addr, enum wmi_peer_param param_id, u32 param_value) { struct wmi_peer_set_param_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_peer_set_param_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->param_id = __cpu_to_le32(param_id); cmd->param_value = __cpu_to_le32(param_value); ether_addr_copy(cmd->peer_macaddr.addr, peer_addr); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi vdev %d peer 0x%pM set param %d value %d\n", vdev_id, peer_addr, param_id, param_value); return skb; } static struct sk_buff *ath10k_wmi_op_gen_gpio_config(struct ath10k *ar, u32 gpio_num, u32 input, u32 pull_type, u32 intr_mode) { struct wmi_gpio_config_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_gpio_config_cmd *)skb->data; cmd->pull_type = __cpu_to_le32(pull_type); cmd->gpio_num = __cpu_to_le32(gpio_num); cmd->input = __cpu_to_le32(input); cmd->intr_mode = __cpu_to_le32(intr_mode); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi gpio_config gpio_num 0x%08x input 0x%08x pull_type 0x%08x intr_mode 0x%08x\n", gpio_num, input, pull_type, intr_mode); return skb; } static struct sk_buff *ath10k_wmi_op_gen_gpio_output(struct ath10k *ar, u32 gpio_num, u32 set) { struct wmi_gpio_output_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_gpio_output_cmd *)skb->data; cmd->gpio_num = __cpu_to_le32(gpio_num); cmd->set = __cpu_to_le32(set); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi gpio_output gpio_num 0x%08x set 0x%08x\n", gpio_num, set); return skb; } static struct sk_buff * ath10k_wmi_op_gen_set_psmode(struct ath10k *ar, u32 vdev_id, enum wmi_sta_ps_mode psmode) { struct wmi_sta_powersave_mode_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_sta_powersave_mode_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->sta_ps_mode = __cpu_to_le32(psmode); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi set powersave id 0x%x mode %d\n", vdev_id, psmode); return skb; } static struct sk_buff * ath10k_wmi_op_gen_set_sta_ps(struct ath10k *ar, u32 vdev_id, enum wmi_sta_powersave_param param_id, u32 value) { struct wmi_sta_powersave_param_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_sta_powersave_param_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->param_id = __cpu_to_le32(param_id); cmd->param_value = __cpu_to_le32(value); ath10k_dbg(ar, ATH10K_DBG_STA, "wmi sta ps param vdev_id 0x%x param %d value %d\n", vdev_id, param_id, value); return skb; } static struct sk_buff * ath10k_wmi_op_gen_set_ap_ps(struct ath10k *ar, u32 vdev_id, const u8 *mac, enum wmi_ap_ps_peer_param param_id, u32 value) { struct wmi_ap_ps_peer_cmd *cmd; struct sk_buff *skb; if (!mac) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_ap_ps_peer_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->param_id = __cpu_to_le32(param_id); cmd->param_value = __cpu_to_le32(value); ether_addr_copy(cmd->peer_macaddr.addr, mac); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi ap ps param vdev_id 0x%X param %d value %d mac_addr %pM\n", vdev_id, param_id, value, mac); return skb; } static struct sk_buff * ath10k_wmi_op_gen_scan_chan_list(struct ath10k *ar, const struct wmi_scan_chan_list_arg *arg) { struct wmi_scan_chan_list_cmd *cmd; struct sk_buff *skb; struct wmi_channel_arg *ch; struct wmi_channel *ci; int i; skb = ath10k_wmi_alloc_skb(ar, struct_size(cmd, chan_info, arg->n_channels)); if (!skb) return ERR_PTR(-EINVAL); cmd = (struct wmi_scan_chan_list_cmd *)skb->data; cmd->num_scan_chans = __cpu_to_le32(arg->n_channels); for (i = 0; i < arg->n_channels; i++) { ch = &arg->channels[i]; ci = &cmd->chan_info[i]; ath10k_wmi_put_wmi_channel(ar, ci, ch); } return skb; } static void ath10k_wmi_peer_assoc_fill(struct ath10k *ar, void *buf, const struct wmi_peer_assoc_complete_arg *arg) { struct wmi_common_peer_assoc_complete_cmd *cmd = buf; cmd->vdev_id = __cpu_to_le32(arg->vdev_id); cmd->peer_new_assoc = __cpu_to_le32(arg->peer_reassoc ? 0 : 1); cmd->peer_associd = __cpu_to_le32(arg->peer_aid); cmd->peer_flags = __cpu_to_le32(arg->peer_flags); cmd->peer_caps = __cpu_to_le32(arg->peer_caps); cmd->peer_listen_intval = __cpu_to_le32(arg->peer_listen_intval); cmd->peer_ht_caps = __cpu_to_le32(arg->peer_ht_caps); cmd->peer_max_mpdu = __cpu_to_le32(arg->peer_max_mpdu); cmd->peer_mpdu_density = __cpu_to_le32(arg->peer_mpdu_density); cmd->peer_rate_caps = __cpu_to_le32(arg->peer_rate_caps); cmd->peer_nss = __cpu_to_le32(arg->peer_num_spatial_streams); cmd->peer_vht_caps = __cpu_to_le32(arg->peer_vht_caps); cmd->peer_phymode = __cpu_to_le32(arg->peer_phymode); ether_addr_copy(cmd->peer_macaddr.addr, arg->addr); cmd->peer_legacy_rates.num_rates = __cpu_to_le32(arg->peer_legacy_rates.num_rates); memcpy(cmd->peer_legacy_rates.rates, arg->peer_legacy_rates.rates, arg->peer_legacy_rates.num_rates); cmd->peer_ht_rates.num_rates = __cpu_to_le32(arg->peer_ht_rates.num_rates); memcpy(cmd->peer_ht_rates.rates, arg->peer_ht_rates.rates, arg->peer_ht_rates.num_rates); cmd->peer_vht_rates.rx_max_rate = __cpu_to_le32(arg->peer_vht_rates.rx_max_rate); cmd->peer_vht_rates.rx_mcs_set = __cpu_to_le32(arg->peer_vht_rates.rx_mcs_set); cmd->peer_vht_rates.tx_max_rate = __cpu_to_le32(arg->peer_vht_rates.tx_max_rate); cmd->peer_vht_rates.tx_mcs_set = __cpu_to_le32(arg->peer_vht_rates.tx_mcs_set); } static void ath10k_wmi_peer_assoc_fill_main(struct ath10k *ar, void *buf, const struct wmi_peer_assoc_complete_arg *arg) { struct wmi_main_peer_assoc_complete_cmd *cmd = buf; ath10k_wmi_peer_assoc_fill(ar, buf, arg); memset(cmd->peer_ht_info, 0, sizeof(cmd->peer_ht_info)); } static void ath10k_wmi_peer_assoc_fill_10_1(struct ath10k *ar, void *buf, const struct wmi_peer_assoc_complete_arg *arg) { ath10k_wmi_peer_assoc_fill(ar, buf, arg); } static void ath10k_wmi_peer_assoc_fill_10_2(struct ath10k *ar, void *buf, const struct wmi_peer_assoc_complete_arg *arg) { struct wmi_10_2_peer_assoc_complete_cmd *cmd = buf; int max_mcs, max_nss; u32 info0; /* TODO: Is using max values okay with firmware? */ max_mcs = 0xf; max_nss = 0xf; info0 = SM(max_mcs, WMI_PEER_ASSOC_INFO0_MAX_MCS_IDX) | SM(max_nss, WMI_PEER_ASSOC_INFO0_MAX_NSS); ath10k_wmi_peer_assoc_fill(ar, buf, arg); cmd->info0 = __cpu_to_le32(info0); } static void ath10k_wmi_peer_assoc_fill_10_4(struct ath10k *ar, void *buf, const struct wmi_peer_assoc_complete_arg *arg) { struct wmi_10_4_peer_assoc_complete_cmd *cmd = buf; ath10k_wmi_peer_assoc_fill_10_2(ar, buf, arg); cmd->peer_bw_rxnss_override = __cpu_to_le32(arg->peer_bw_rxnss_override); } static int ath10k_wmi_peer_assoc_check_arg(const struct wmi_peer_assoc_complete_arg *arg) { if (arg->peer_mpdu_density > 16) return -EINVAL; if (arg->peer_legacy_rates.num_rates > MAX_SUPPORTED_RATES) return -EINVAL; if (arg->peer_ht_rates.num_rates > MAX_SUPPORTED_RATES) return -EINVAL; return 0; } static struct sk_buff * ath10k_wmi_op_gen_peer_assoc(struct ath10k *ar, const struct wmi_peer_assoc_complete_arg *arg) { size_t len = sizeof(struct wmi_main_peer_assoc_complete_cmd); struct sk_buff *skb; int ret; ret = ath10k_wmi_peer_assoc_check_arg(arg); if (ret) return ERR_PTR(ret); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); ath10k_wmi_peer_assoc_fill_main(ar, skb->data, arg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer assoc vdev %d addr %pM (%s)\n", arg->vdev_id, arg->addr, arg->peer_reassoc ? "reassociate" : "new"); return skb; } static struct sk_buff * ath10k_wmi_10_1_op_gen_peer_assoc(struct ath10k *ar, const struct wmi_peer_assoc_complete_arg *arg) { size_t len = sizeof(struct wmi_10_1_peer_assoc_complete_cmd); struct sk_buff *skb; int ret; ret = ath10k_wmi_peer_assoc_check_arg(arg); if (ret) return ERR_PTR(ret); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); ath10k_wmi_peer_assoc_fill_10_1(ar, skb->data, arg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer assoc vdev %d addr %pM (%s)\n", arg->vdev_id, arg->addr, arg->peer_reassoc ? "reassociate" : "new"); return skb; } static struct sk_buff * ath10k_wmi_10_2_op_gen_peer_assoc(struct ath10k *ar, const struct wmi_peer_assoc_complete_arg *arg) { size_t len = sizeof(struct wmi_10_2_peer_assoc_complete_cmd); struct sk_buff *skb; int ret; ret = ath10k_wmi_peer_assoc_check_arg(arg); if (ret) return ERR_PTR(ret); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); ath10k_wmi_peer_assoc_fill_10_2(ar, skb->data, arg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer assoc vdev %d addr %pM (%s)\n", arg->vdev_id, arg->addr, arg->peer_reassoc ? "reassociate" : "new"); return skb; } static struct sk_buff * ath10k_wmi_10_4_op_gen_peer_assoc(struct ath10k *ar, const struct wmi_peer_assoc_complete_arg *arg) { size_t len = sizeof(struct wmi_10_4_peer_assoc_complete_cmd); struct sk_buff *skb; int ret; ret = ath10k_wmi_peer_assoc_check_arg(arg); if (ret) return ERR_PTR(ret); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); ath10k_wmi_peer_assoc_fill_10_4(ar, skb->data, arg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer assoc vdev %d addr %pM (%s)\n", arg->vdev_id, arg->addr, arg->peer_reassoc ? "reassociate" : "new"); return skb; } static struct sk_buff * ath10k_wmi_10_2_op_gen_pdev_get_temperature(struct ath10k *ar) { struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, 0); if (!skb) return ERR_PTR(-ENOMEM); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev get temperature\n"); return skb; } static struct sk_buff * ath10k_wmi_10_2_op_gen_pdev_bss_chan_info(struct ath10k *ar, enum wmi_bss_survey_req_type type) { struct wmi_pdev_chan_info_req_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_chan_info_req_cmd *)skb->data; cmd->type = __cpu_to_le32(type); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev bss info request type %d\n", type); return skb; } /* This function assumes the beacon is already DMA mapped */ static struct sk_buff * ath10k_wmi_op_gen_beacon_dma(struct ath10k *ar, u32 vdev_id, const void *bcn, size_t bcn_len, u32 bcn_paddr, bool dtim_zero, bool deliver_cab) { struct wmi_bcn_tx_ref_cmd *cmd; struct sk_buff *skb; struct ieee80211_hdr *hdr; u16 fc; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); hdr = (struct ieee80211_hdr *)bcn; fc = le16_to_cpu(hdr->frame_control); cmd = (struct wmi_bcn_tx_ref_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->data_len = __cpu_to_le32(bcn_len); cmd->data_ptr = __cpu_to_le32(bcn_paddr); cmd->msdu_id = 0; cmd->frame_control = __cpu_to_le32(fc); cmd->flags = 0; cmd->antenna_mask = __cpu_to_le32(WMI_BCN_TX_REF_DEF_ANTENNA); if (dtim_zero) cmd->flags |= __cpu_to_le32(WMI_BCN_TX_REF_FLAG_DTIM_ZERO); if (deliver_cab) cmd->flags |= __cpu_to_le32(WMI_BCN_TX_REF_FLAG_DELIVER_CAB); return skb; } void ath10k_wmi_set_wmm_param(struct wmi_wmm_params *params, const struct wmi_wmm_params_arg *arg) { params->cwmin = __cpu_to_le32(arg->cwmin); params->cwmax = __cpu_to_le32(arg->cwmax); params->aifs = __cpu_to_le32(arg->aifs); params->txop = __cpu_to_le32(arg->txop); params->acm = __cpu_to_le32(arg->acm); params->no_ack = __cpu_to_le32(arg->no_ack); } static struct sk_buff * ath10k_wmi_op_gen_pdev_set_wmm(struct ath10k *ar, const struct wmi_wmm_params_all_arg *arg) { struct wmi_pdev_set_wmm_params *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_wmm_params *)skb->data; ath10k_wmi_set_wmm_param(&cmd->ac_be, &arg->ac_be); ath10k_wmi_set_wmm_param(&cmd->ac_bk, &arg->ac_bk); ath10k_wmi_set_wmm_param(&cmd->ac_vi, &arg->ac_vi); ath10k_wmi_set_wmm_param(&cmd->ac_vo, &arg->ac_vo); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev set wmm params\n"); return skb; } static struct sk_buff * ath10k_wmi_op_gen_request_stats(struct ath10k *ar, u32 stats_mask) { struct wmi_request_stats_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_request_stats_cmd *)skb->data; cmd->stats_id = __cpu_to_le32(stats_mask); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi request stats 0x%08x\n", stats_mask); return skb; } static struct sk_buff * ath10k_wmi_op_gen_force_fw_hang(struct ath10k *ar, enum wmi_force_fw_hang_type type, u32 delay_ms) { struct wmi_force_fw_hang_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_force_fw_hang_cmd *)skb->data; cmd->type = __cpu_to_le32(type); cmd->delay_ms = __cpu_to_le32(delay_ms); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi force fw hang %d delay %d\n", type, delay_ms); return skb; } static struct sk_buff * ath10k_wmi_op_gen_dbglog_cfg(struct ath10k *ar, u64 module_enable, u32 log_level) { struct wmi_dbglog_cfg_cmd *cmd; struct sk_buff *skb; u32 cfg; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_dbglog_cfg_cmd *)skb->data; if (module_enable) { cfg = SM(log_level, ATH10K_DBGLOG_CFG_LOG_LVL); } else { /* set back defaults, all modules with WARN level */ cfg = SM(ATH10K_DBGLOG_LEVEL_WARN, ATH10K_DBGLOG_CFG_LOG_LVL); module_enable = ~0; } cmd->module_enable = __cpu_to_le32(module_enable); cmd->module_valid = __cpu_to_le32(~0); cmd->config_enable = __cpu_to_le32(cfg); cmd->config_valid = __cpu_to_le32(ATH10K_DBGLOG_CFG_LOG_LVL_MASK); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi dbglog cfg modules %08x %08x config %08x %08x\n", __le32_to_cpu(cmd->module_enable), __le32_to_cpu(cmd->module_valid), __le32_to_cpu(cmd->config_enable), __le32_to_cpu(cmd->config_valid)); return skb; } static struct sk_buff * ath10k_wmi_10_4_op_gen_dbglog_cfg(struct ath10k *ar, u64 module_enable, u32 log_level) { struct wmi_10_4_dbglog_cfg_cmd *cmd; struct sk_buff *skb; u32 cfg; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_10_4_dbglog_cfg_cmd *)skb->data; if (module_enable) { cfg = SM(log_level, ATH10K_DBGLOG_CFG_LOG_LVL); } else { /* set back defaults, all modules with WARN level */ cfg = SM(ATH10K_DBGLOG_LEVEL_WARN, ATH10K_DBGLOG_CFG_LOG_LVL); module_enable = ~0; } cmd->module_enable = __cpu_to_le64(module_enable); cmd->module_valid = __cpu_to_le64(~0); cmd->config_enable = __cpu_to_le32(cfg); cmd->config_valid = __cpu_to_le32(ATH10K_DBGLOG_CFG_LOG_LVL_MASK); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi dbglog cfg modules 0x%016llx 0x%016llx config %08x %08x\n", __le64_to_cpu(cmd->module_enable), __le64_to_cpu(cmd->module_valid), __le32_to_cpu(cmd->config_enable), __le32_to_cpu(cmd->config_valid)); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pktlog_enable(struct ath10k *ar, u32 ev_bitmap) { struct wmi_pdev_pktlog_enable_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); ev_bitmap &= ATH10K_PKTLOG_ANY; cmd = (struct wmi_pdev_pktlog_enable_cmd *)skb->data; cmd->ev_bitmap = __cpu_to_le32(ev_bitmap); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi enable pktlog filter 0x%08x\n", ev_bitmap); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pktlog_disable(struct ath10k *ar) { struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, 0); if (!skb) return ERR_PTR(-ENOMEM); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi disable pktlog\n"); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pdev_set_quiet_mode(struct ath10k *ar, u32 period, u32 duration, u32 next_offset, u32 enabled) { struct wmi_pdev_set_quiet_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_quiet_cmd *)skb->data; cmd->period = __cpu_to_le32(period); cmd->duration = __cpu_to_le32(duration); cmd->next_start = __cpu_to_le32(next_offset); cmd->enabled = __cpu_to_le32(enabled); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi quiet param: period %u duration %u enabled %d\n", period, duration, enabled); return skb; } static struct sk_buff * ath10k_wmi_op_gen_addba_clear_resp(struct ath10k *ar, u32 vdev_id, const u8 *mac) { struct wmi_addba_clear_resp_cmd *cmd; struct sk_buff *skb; if (!mac) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_addba_clear_resp_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, mac); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi addba clear resp vdev_id 0x%X mac_addr %pM\n", vdev_id, mac); return skb; } static struct sk_buff * ath10k_wmi_op_gen_addba_send(struct ath10k *ar, u32 vdev_id, const u8 *mac, u32 tid, u32 buf_size) { struct wmi_addba_send_cmd *cmd; struct sk_buff *skb; if (!mac) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_addba_send_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, mac); cmd->tid = __cpu_to_le32(tid); cmd->buffersize = __cpu_to_le32(buf_size); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi addba send vdev_id 0x%X mac_addr %pM tid %u bufsize %u\n", vdev_id, mac, tid, buf_size); return skb; } static struct sk_buff * ath10k_wmi_op_gen_addba_set_resp(struct ath10k *ar, u32 vdev_id, const u8 *mac, u32 tid, u32 status) { struct wmi_addba_setresponse_cmd *cmd; struct sk_buff *skb; if (!mac) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_addba_setresponse_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, mac); cmd->tid = __cpu_to_le32(tid); cmd->statuscode = __cpu_to_le32(status); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi addba set resp vdev_id 0x%X mac_addr %pM tid %u status %u\n", vdev_id, mac, tid, status); return skb; } static struct sk_buff * ath10k_wmi_op_gen_delba_send(struct ath10k *ar, u32 vdev_id, const u8 *mac, u32 tid, u32 initiator, u32 reason) { struct wmi_delba_send_cmd *cmd; struct sk_buff *skb; if (!mac) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_delba_send_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, mac); cmd->tid = __cpu_to_le32(tid); cmd->initiator = __cpu_to_le32(initiator); cmd->reasoncode = __cpu_to_le32(reason); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi delba send vdev_id 0x%X mac_addr %pM tid %u initiator %u reason %u\n", vdev_id, mac, tid, initiator, reason); return skb; } static struct sk_buff * ath10k_wmi_10_2_4_op_gen_pdev_get_tpc_config(struct ath10k *ar, u32 param) { struct wmi_pdev_get_tpc_config_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_get_tpc_config_cmd *)skb->data; cmd->param = __cpu_to_le32(param); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev get tpc config param %d\n", param); return skb; } static void ath10k_wmi_fw_pdev_base_stats_fill(const struct ath10k_fw_stats_pdev *pdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s\n", "ath10k PDEV stats"); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Channel noise floor", pdev->ch_noise_floor); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "Channel TX power", pdev->chan_tx_power); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "TX frame count", pdev->tx_frame_count); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "RX frame count", pdev->rx_frame_count); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "RX clear count", pdev->rx_clear_count); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "Cycle count", pdev->cycle_count); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "PHY error count", pdev->phy_err_count); *length = len; } static void ath10k_wmi_fw_pdev_extra_stats_fill(const struct ath10k_fw_stats_pdev *pdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "RTS bad count", pdev->rts_bad); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "RTS good count", pdev->rts_good); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "FCS bad count", pdev->fcs_bad); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "No beacon count", pdev->no_beacons); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "MIB int count", pdev->mib_int_count); len += scnprintf(buf + len, buf_len - len, "\n"); *length = len; } static void ath10k_wmi_fw_pdev_tx_stats_fill(const struct ath10k_fw_stats_pdev *pdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "\n%30s\n", "ath10k PDEV TX stats"); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "HTT cookies queued", pdev->comp_queued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "HTT cookies disp.", pdev->comp_delivered); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MSDU queued", pdev->msdu_enqued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDU queued", pdev->mpdu_enqued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MSDUs dropped", pdev->wmm_drop); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Local enqued", pdev->local_enqued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Local freed", pdev->local_freed); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "HW queued", pdev->hw_queued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PPDUs reaped", pdev->hw_reaped); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Num underruns", pdev->underrun); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PPDUs cleaned", pdev->tx_abort); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs requeued", pdev->mpdus_requeued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Excessive retries", pdev->tx_ko); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "HW rate", pdev->data_rc); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Sched self triggers", pdev->self_triggers); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Dropped due to SW retries", pdev->sw_retry_failure); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Illegal rate phy errors", pdev->illgl_rate_phy_err); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Pdev continuous xretry", pdev->pdev_cont_xretry); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "TX timeout", pdev->pdev_tx_timeout); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PDEV resets", pdev->pdev_resets); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PHY underrun", pdev->phy_underrun); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDU is more than txop limit", pdev->txop_ovf); *length = len; } static void ath10k_wmi_fw_pdev_rx_stats_fill(const struct ath10k_fw_stats_pdev *pdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "\n%30s\n", "ath10k PDEV RX stats"); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Mid PPDU route change", pdev->mid_ppdu_route_change); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Tot. number of statuses", pdev->status_rcvd); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Extra frags on rings 0", pdev->r0_frags); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Extra frags on rings 1", pdev->r1_frags); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Extra frags on rings 2", pdev->r2_frags); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Extra frags on rings 3", pdev->r3_frags); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MSDUs delivered to HTT", pdev->htt_msdus); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs delivered to HTT", pdev->htt_mpdus); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MSDUs delivered to stack", pdev->loc_msdus); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs delivered to stack", pdev->loc_mpdus); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Oversized AMSDUs", pdev->oversize_amsdu); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PHY errors", pdev->phy_errs); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PHY errors drops", pdev->phy_err_drop); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDU errors (FCS, MIC, ENC)", pdev->mpdu_errs); *length = len; } static void ath10k_wmi_fw_vdev_stats_fill(const struct ath10k_fw_stats_vdev *vdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; int i; len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "vdev id", vdev->vdev_id); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "beacon snr", vdev->beacon_snr); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "data snr", vdev->data_snr); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num rx frames", vdev->num_rx_frames); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num rts fail", vdev->num_rts_fail); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num rts success", vdev->num_rts_success); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num rx err", vdev->num_rx_err); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num rx discard", vdev->num_rx_discard); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num tx not acked", vdev->num_tx_not_acked); for (i = 0 ; i < ARRAY_SIZE(vdev->num_tx_frames); i++) len += scnprintf(buf + len, buf_len - len, "%25s [%02d] %u\n", "num tx frames", i, vdev->num_tx_frames[i]); for (i = 0 ; i < ARRAY_SIZE(vdev->num_tx_frames_retries); i++) len += scnprintf(buf + len, buf_len - len, "%25s [%02d] %u\n", "num tx frames retries", i, vdev->num_tx_frames_retries[i]); for (i = 0 ; i < ARRAY_SIZE(vdev->num_tx_frames_failures); i++) len += scnprintf(buf + len, buf_len - len, "%25s [%02d] %u\n", "num tx frames failures", i, vdev->num_tx_frames_failures[i]); for (i = 0 ; i < ARRAY_SIZE(vdev->tx_rate_history); i++) len += scnprintf(buf + len, buf_len - len, "%25s [%02d] 0x%08x\n", "tx rate history", i, vdev->tx_rate_history[i]); for (i = 0 ; i < ARRAY_SIZE(vdev->beacon_rssi_history); i++) len += scnprintf(buf + len, buf_len - len, "%25s [%02d] %u\n", "beacon rssi history", i, vdev->beacon_rssi_history[i]); len += scnprintf(buf + len, buf_len - len, "\n"); *length = len; } static void ath10k_wmi_fw_peer_stats_fill(const struct ath10k_fw_stats_peer *peer, char *buf, u32 *length, bool extended_peer) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "%30s %pM\n", "Peer MAC address", peer->peer_macaddr); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "Peer RSSI", peer->peer_rssi); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "Peer TX rate", peer->peer_tx_rate); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "Peer RX rate", peer->peer_rx_rate); if (!extended_peer) len += scnprintf(buf + len, buf_len - len, "%30s %llu\n", "Peer RX duration", peer->rx_duration); len += scnprintf(buf + len, buf_len - len, "\n"); *length = len; } static void ath10k_wmi_fw_extd_peer_stats_fill(const struct ath10k_fw_extd_stats_peer *peer, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "%30s %pM\n", "Peer MAC address", peer->peer_macaddr); len += scnprintf(buf + len, buf_len - len, "%30s %llu\n", "Peer RX duration", peer->rx_duration); } void ath10k_wmi_main_op_fw_stats_fill(struct ath10k *ar, struct ath10k_fw_stats *fw_stats, char *buf) { u32 len = 0; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; const struct ath10k_fw_stats_pdev *pdev; const struct ath10k_fw_stats_vdev *vdev; const struct ath10k_fw_stats_peer *peer; size_t num_peers; size_t num_vdevs; spin_lock_bh(&ar->data_lock); pdev = list_first_entry_or_null(&fw_stats->pdevs, struct ath10k_fw_stats_pdev, list); if (!pdev) { ath10k_warn(ar, "failed to get pdev stats\n"); goto unlock; } num_peers = list_count_nodes(&fw_stats->peers); num_vdevs = list_count_nodes(&fw_stats->vdevs); ath10k_wmi_fw_pdev_base_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_tx_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_rx_stats_fill(pdev, buf, &len); len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k VDEV stats", num_vdevs); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(vdev, &fw_stats->vdevs, list) { ath10k_wmi_fw_vdev_stats_fill(vdev, buf, &len); } len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k PEER stats", num_peers); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(peer, &fw_stats->peers, list) { ath10k_wmi_fw_peer_stats_fill(peer, buf, &len, fw_stats->extended); } unlock: spin_unlock_bh(&ar->data_lock); if (len >= buf_len) buf[len - 1] = 0; else buf[len] = 0; } void ath10k_wmi_10x_op_fw_stats_fill(struct ath10k *ar, struct ath10k_fw_stats *fw_stats, char *buf) { unsigned int len = 0; unsigned int buf_len = ATH10K_FW_STATS_BUF_SIZE; const struct ath10k_fw_stats_pdev *pdev; const struct ath10k_fw_stats_vdev *vdev; const struct ath10k_fw_stats_peer *peer; size_t num_peers; size_t num_vdevs; spin_lock_bh(&ar->data_lock); pdev = list_first_entry_or_null(&fw_stats->pdevs, struct ath10k_fw_stats_pdev, list); if (!pdev) { ath10k_warn(ar, "failed to get pdev stats\n"); goto unlock; } num_peers = list_count_nodes(&fw_stats->peers); num_vdevs = list_count_nodes(&fw_stats->vdevs); ath10k_wmi_fw_pdev_base_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_extra_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_tx_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_rx_stats_fill(pdev, buf, &len); len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k VDEV stats", num_vdevs); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(vdev, &fw_stats->vdevs, list) { ath10k_wmi_fw_vdev_stats_fill(vdev, buf, &len); } len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k PEER stats", num_peers); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(peer, &fw_stats->peers, list) { ath10k_wmi_fw_peer_stats_fill(peer, buf, &len, fw_stats->extended); } unlock: spin_unlock_bh(&ar->data_lock); if (len >= buf_len) buf[len - 1] = 0; else buf[len] = 0; } static struct sk_buff * ath10k_wmi_op_gen_pdev_enable_adaptive_cca(struct ath10k *ar, u8 enable, u32 detect_level, u32 detect_margin) { struct wmi_pdev_set_adaptive_cca_params *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_adaptive_cca_params *)skb->data; cmd->enable = __cpu_to_le32(enable); cmd->cca_detect_level = __cpu_to_le32(detect_level); cmd->cca_detect_margin = __cpu_to_le32(detect_margin); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev set adaptive cca params enable:%d detection level:%d detection margin:%d\n", enable, detect_level, detect_margin); return skb; } static void ath10k_wmi_fw_vdev_stats_extd_fill(const struct ath10k_fw_stats_vdev_extd *vdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; u32 val; len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "vdev id", vdev->vdev_id); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "ppdu aggr count", vdev->ppdu_aggr_cnt); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "ppdu noack", vdev->ppdu_noack); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "mpdu queued", vdev->mpdu_queued); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "ppdu nonaggr count", vdev->ppdu_nonaggr_cnt); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "mpdu sw requeued", vdev->mpdu_sw_requeued); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "mpdu success retry", vdev->mpdu_suc_retry); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "mpdu success multitry", vdev->mpdu_suc_multitry); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "mpdu fail retry", vdev->mpdu_fail_retry); val = vdev->tx_ftm_suc; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "tx ftm success", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->tx_ftm_suc_retry; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "tx ftm success retry", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->tx_ftm_fail; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "tx ftm fail", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->rx_ftmr_cnt; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "rx ftm request count", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->rx_ftmr_dup_cnt; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "rx ftm request dup count", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->rx_iftmr_cnt; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "rx initial ftm req count", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->rx_iftmr_dup_cnt; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "rx initial ftm req dup cnt", MS(val, WMI_VDEV_STATS_FTM_COUNT)); len += scnprintf(buf + len, buf_len - len, "\n"); *length = len; } void ath10k_wmi_10_4_op_fw_stats_fill(struct ath10k *ar, struct ath10k_fw_stats *fw_stats, char *buf) { u32 len = 0; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; const struct ath10k_fw_stats_pdev *pdev; const struct ath10k_fw_stats_vdev_extd *vdev; const struct ath10k_fw_stats_peer *peer; const struct ath10k_fw_extd_stats_peer *extd_peer; size_t num_peers; size_t num_vdevs; spin_lock_bh(&ar->data_lock); pdev = list_first_entry_or_null(&fw_stats->pdevs, struct ath10k_fw_stats_pdev, list); if (!pdev) { ath10k_warn(ar, "failed to get pdev stats\n"); goto unlock; } num_peers = list_count_nodes(&fw_stats->peers); num_vdevs = list_count_nodes(&fw_stats->vdevs); ath10k_wmi_fw_pdev_base_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_extra_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_tx_stats_fill(pdev, buf, &len); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "HW paused", pdev->hw_paused); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Seqs posted", pdev->seq_posted); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Seqs failed queueing", pdev->seq_failed_queueing); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Seqs completed", pdev->seq_completed); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Seqs restarted", pdev->seq_restarted); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MU Seqs posted", pdev->mu_seq_posted); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs SW flushed", pdev->mpdus_sw_flush); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs HW filtered", pdev->mpdus_hw_filter); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs truncated", pdev->mpdus_truncated); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs receive no ACK", pdev->mpdus_ack_failed); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs expired", pdev->mpdus_expired); ath10k_wmi_fw_pdev_rx_stats_fill(pdev, buf, &len); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Num Rx Overflow errors", pdev->rx_ovfl_errs); len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k VDEV stats", num_vdevs); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(vdev, &fw_stats->vdevs, list) { ath10k_wmi_fw_vdev_stats_extd_fill(vdev, buf, &len); } len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k PEER stats", num_peers); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(peer, &fw_stats->peers, list) { ath10k_wmi_fw_peer_stats_fill(peer, buf, &len, fw_stats->extended); } if (fw_stats->extended) { list_for_each_entry(extd_peer, &fw_stats->peers_extd, list) { ath10k_wmi_fw_extd_peer_stats_fill(extd_peer, buf, &len); } } unlock: spin_unlock_bh(&ar->data_lock); if (len >= buf_len) buf[len - 1] = 0; else buf[len] = 0; } int ath10k_wmi_op_get_vdev_subtype(struct ath10k *ar, enum wmi_vdev_subtype subtype) { switch (subtype) { case WMI_VDEV_SUBTYPE_NONE: return WMI_VDEV_SUBTYPE_LEGACY_NONE; case WMI_VDEV_SUBTYPE_P2P_DEVICE: return WMI_VDEV_SUBTYPE_LEGACY_P2P_DEV; case WMI_VDEV_SUBTYPE_P2P_CLIENT: return WMI_VDEV_SUBTYPE_LEGACY_P2P_CLI; case WMI_VDEV_SUBTYPE_P2P_GO: return WMI_VDEV_SUBTYPE_LEGACY_P2P_GO; case WMI_VDEV_SUBTYPE_PROXY_STA: return WMI_VDEV_SUBTYPE_LEGACY_PROXY_STA; case WMI_VDEV_SUBTYPE_MESH_11S: case WMI_VDEV_SUBTYPE_MESH_NON_11S: return -EOPNOTSUPP; } return -EOPNOTSUPP; } static int ath10k_wmi_10_2_4_op_get_vdev_subtype(struct ath10k *ar, enum wmi_vdev_subtype subtype) { switch (subtype) { case WMI_VDEV_SUBTYPE_NONE: return WMI_VDEV_SUBTYPE_10_2_4_NONE; case WMI_VDEV_SUBTYPE_P2P_DEVICE: return WMI_VDEV_SUBTYPE_10_2_4_P2P_DEV; case WMI_VDEV_SUBTYPE_P2P_CLIENT: return WMI_VDEV_SUBTYPE_10_2_4_P2P_CLI; case WMI_VDEV_SUBTYPE_P2P_GO: return WMI_VDEV_SUBTYPE_10_2_4_P2P_GO; case WMI_VDEV_SUBTYPE_PROXY_STA: return WMI_VDEV_SUBTYPE_10_2_4_PROXY_STA; case WMI_VDEV_SUBTYPE_MESH_11S: return WMI_VDEV_SUBTYPE_10_2_4_MESH_11S; case WMI_VDEV_SUBTYPE_MESH_NON_11S: return -EOPNOTSUPP; } return -EOPNOTSUPP; } static int ath10k_wmi_10_4_op_get_vdev_subtype(struct ath10k *ar, enum wmi_vdev_subtype subtype) { switch (subtype) { case WMI_VDEV_SUBTYPE_NONE: return WMI_VDEV_SUBTYPE_10_4_NONE; case WMI_VDEV_SUBTYPE_P2P_DEVICE: return WMI_VDEV_SUBTYPE_10_4_P2P_DEV; case WMI_VDEV_SUBTYPE_P2P_CLIENT: return WMI_VDEV_SUBTYPE_10_4_P2P_CLI; case WMI_VDEV_SUBTYPE_P2P_GO: return WMI_VDEV_SUBTYPE_10_4_P2P_GO; case WMI_VDEV_SUBTYPE_PROXY_STA: return WMI_VDEV_SUBTYPE_10_4_PROXY_STA; case WMI_VDEV_SUBTYPE_MESH_11S: return WMI_VDEV_SUBTYPE_10_4_MESH_11S; case WMI_VDEV_SUBTYPE_MESH_NON_11S: return WMI_VDEV_SUBTYPE_10_4_MESH_NON_11S; } return -EOPNOTSUPP; } static struct sk_buff * ath10k_wmi_10_4_ext_resource_config(struct ath10k *ar, enum wmi_host_platform_type type, u32 fw_feature_bitmap) { struct wmi_ext_resource_config_10_4_cmd *cmd; struct sk_buff *skb; u32 num_tdls_sleep_sta = 0; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); if (test_bit(WMI_SERVICE_TDLS_UAPSD_SLEEP_STA, ar->wmi.svc_map)) num_tdls_sleep_sta = TARGET_10_4_NUM_TDLS_SLEEP_STA; cmd = (struct wmi_ext_resource_config_10_4_cmd *)skb->data; cmd->host_platform_config = __cpu_to_le32(type); cmd->fw_feature_bitmap = __cpu_to_le32(fw_feature_bitmap); cmd->wlan_gpio_priority = __cpu_to_le32(ar->coex_gpio_pin); cmd->coex_version = __cpu_to_le32(WMI_NO_COEX_VERSION_SUPPORT); cmd->coex_gpio_pin1 = __cpu_to_le32(-1); cmd->coex_gpio_pin2 = __cpu_to_le32(-1); cmd->coex_gpio_pin3 = __cpu_to_le32(-1); cmd->num_tdls_vdevs = __cpu_to_le32(TARGET_10_4_NUM_TDLS_VDEVS); cmd->num_tdls_conn_table_entries = __cpu_to_le32(20); cmd->max_tdls_concurrent_sleep_sta = __cpu_to_le32(num_tdls_sleep_sta); cmd->max_tdls_concurrent_buffer_sta = __cpu_to_le32(TARGET_10_4_NUM_TDLS_BUFFER_STA); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi ext resource config host type %d firmware feature bitmap %08x\n", type, fw_feature_bitmap); return skb; } static struct sk_buff * ath10k_wmi_10_4_gen_update_fw_tdls_state(struct ath10k *ar, u32 vdev_id, enum wmi_tdls_state state) { struct wmi_10_4_tdls_set_state_cmd *cmd; struct sk_buff *skb; u32 options = 0; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); if (test_bit(WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY, ar->wmi.svc_map) && state == WMI_TDLS_ENABLE_ACTIVE) state = WMI_TDLS_ENABLE_PASSIVE; if (test_bit(WMI_SERVICE_TDLS_UAPSD_BUFFER_STA, ar->wmi.svc_map)) options |= WMI_TDLS_BUFFER_STA_EN; cmd = (struct wmi_10_4_tdls_set_state_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->state = __cpu_to_le32(state); cmd->notification_interval_ms = __cpu_to_le32(5000); cmd->tx_discovery_threshold = __cpu_to_le32(100); cmd->tx_teardown_threshold = __cpu_to_le32(5); cmd->rssi_teardown_threshold = __cpu_to_le32(-75); cmd->rssi_delta = __cpu_to_le32(-20); cmd->tdls_options = __cpu_to_le32(options); cmd->tdls_peer_traffic_ind_window = __cpu_to_le32(2); cmd->tdls_peer_traffic_response_timeout_ms = __cpu_to_le32(5000); cmd->tdls_puapsd_mask = __cpu_to_le32(0xf); cmd->tdls_puapsd_inactivity_time_ms = __cpu_to_le32(0); cmd->tdls_puapsd_rx_frame_threshold = __cpu_to_le32(10); cmd->teardown_notification_ms = __cpu_to_le32(10); cmd->tdls_peer_kickout_threshold = __cpu_to_le32(96); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi update fw tdls state %d for vdev %i\n", state, vdev_id); return skb; } static u32 ath10k_wmi_prepare_peer_qos(u8 uapsd_queues, u8 sp) { u32 peer_qos = 0; if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VO) peer_qos |= WMI_TDLS_PEER_QOS_AC_VO; if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VI) peer_qos |= WMI_TDLS_PEER_QOS_AC_VI; if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BK) peer_qos |= WMI_TDLS_PEER_QOS_AC_BK; if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BE) peer_qos |= WMI_TDLS_PEER_QOS_AC_BE; peer_qos |= SM(sp, WMI_TDLS_PEER_SP); return peer_qos; } static struct sk_buff * ath10k_wmi_10_4_op_gen_pdev_get_tpc_table_cmdid(struct ath10k *ar, u32 param) { struct wmi_pdev_get_tpc_table_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_get_tpc_table_cmd *)skb->data; cmd->param = __cpu_to_le32(param); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev get tpc table param:%d\n", param); return skb; } static struct sk_buff * ath10k_wmi_10_4_gen_tdls_peer_update(struct ath10k *ar, const struct wmi_tdls_peer_update_cmd_arg *arg, const struct wmi_tdls_peer_capab_arg *cap, const struct wmi_channel_arg *chan_arg) { struct wmi_10_4_tdls_peer_update_cmd *cmd; struct wmi_tdls_peer_capabilities *peer_cap; struct wmi_channel *chan; struct sk_buff *skb; u32 peer_qos; int len, chan_len; int i; /* tdls peer update cmd has place holder for one channel*/ chan_len = cap->peer_chan_len ? (cap->peer_chan_len - 1) : 0; len = sizeof(*cmd) + chan_len * sizeof(*chan); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_10_4_tdls_peer_update_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(arg->vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, arg->addr); cmd->peer_state = __cpu_to_le32(arg->peer_state); peer_qos = ath10k_wmi_prepare_peer_qos(cap->peer_uapsd_queues, cap->peer_max_sp); peer_cap = &cmd->peer_capab; peer_cap->peer_qos = __cpu_to_le32(peer_qos); peer_cap->buff_sta_support = __cpu_to_le32(cap->buff_sta_support); peer_cap->off_chan_support = __cpu_to_le32(cap->off_chan_support); peer_cap->peer_curr_operclass = __cpu_to_le32(cap->peer_curr_operclass); peer_cap->self_curr_operclass = __cpu_to_le32(cap->self_curr_operclass); peer_cap->peer_chan_len = __cpu_to_le32(cap->peer_chan_len); peer_cap->peer_operclass_len = __cpu_to_le32(cap->peer_operclass_len); for (i = 0; i < WMI_TDLS_MAX_SUPP_OPER_CLASSES; i++) peer_cap->peer_operclass[i] = cap->peer_operclass[i]; peer_cap->is_peer_responder = __cpu_to_le32(cap->is_peer_responder); peer_cap->pref_offchan_num = __cpu_to_le32(cap->pref_offchan_num); peer_cap->pref_offchan_bw = __cpu_to_le32(cap->pref_offchan_bw); for (i = 0; i < cap->peer_chan_len; i++) { chan = (struct wmi_channel *)&peer_cap->peer_chan_list[i]; ath10k_wmi_put_wmi_channel(ar, chan, &chan_arg[i]); } ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi tdls peer update vdev %i state %d n_chans %u\n", arg->vdev_id, arg->peer_state, cap->peer_chan_len); return skb; } static struct sk_buff * ath10k_wmi_10_4_gen_radar_found(struct ath10k *ar, const struct ath10k_radar_found_info *arg) { struct wmi_radar_found_info *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_radar_found_info *)skb->data; cmd->pri_min = __cpu_to_le32(arg->pri_min); cmd->pri_max = __cpu_to_le32(arg->pri_max); cmd->width_min = __cpu_to_le32(arg->width_min); cmd->width_max = __cpu_to_le32(arg->width_max); cmd->sidx_min = __cpu_to_le32(arg->sidx_min); cmd->sidx_max = __cpu_to_le32(arg->sidx_max); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi radar found pri_min %d pri_max %d width_min %d width_max %d sidx_min %d sidx_max %d\n", arg->pri_min, arg->pri_max, arg->width_min, arg->width_max, arg->sidx_min, arg->sidx_max); return skb; } static struct sk_buff * ath10k_wmi_10_4_gen_per_peer_per_tid_cfg(struct ath10k *ar, const struct wmi_per_peer_per_tid_cfg_arg *arg) { struct wmi_peer_per_tid_cfg_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); memset(skb->data, 0, sizeof(*cmd)); cmd = (struct wmi_peer_per_tid_cfg_cmd *)skb->data; cmd->vdev_id = cpu_to_le32(arg->vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, arg->peer_macaddr.addr); cmd->tid = cpu_to_le32(arg->tid); cmd->ack_policy = cpu_to_le32(arg->ack_policy); cmd->aggr_control = cpu_to_le32(arg->aggr_control); cmd->rate_control = cpu_to_le32(arg->rate_ctrl); cmd->retry_count = cpu_to_le32(arg->retry_count); cmd->rcode_flags = cpu_to_le32(arg->rcode_flags); cmd->ext_tid_cfg_bitmap = cpu_to_le32(arg->ext_tid_cfg_bitmap); cmd->rtscts_ctrl = cpu_to_le32(arg->rtscts_ctrl); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi noack tid %d vdev id %d ack_policy %d aggr %u rate_ctrl %u rcflag %u retry_count %d rtscts %d ext_tid_cfg_bitmap %d mac_addr %pM\n", arg->tid, arg->vdev_id, arg->ack_policy, arg->aggr_control, arg->rate_ctrl, arg->rcode_flags, arg->retry_count, arg->rtscts_ctrl, arg->ext_tid_cfg_bitmap, arg->peer_macaddr.addr); return skb; } static struct sk_buff * ath10k_wmi_op_gen_echo(struct ath10k *ar, u32 value) { struct wmi_echo_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_echo_cmd *)skb->data; cmd->value = cpu_to_le32(value); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi echo value 0x%08x\n", value); return skb; } int ath10k_wmi_barrier(struct ath10k *ar) { int ret; int time_left; spin_lock_bh(&ar->data_lock); reinit_completion(&ar->wmi.barrier); spin_unlock_bh(&ar->data_lock); ret = ath10k_wmi_echo(ar, ATH10K_WMI_BARRIER_ECHO_ID); if (ret) { ath10k_warn(ar, "failed to submit wmi echo: %d\n", ret); return ret; } time_left = wait_for_completion_timeout(&ar->wmi.barrier, ATH10K_WMI_BARRIER_TIMEOUT_HZ); if (!time_left) return -ETIMEDOUT; return 0; } static struct sk_buff * ath10k_wmi_10_2_4_op_gen_bb_timing(struct ath10k *ar, const struct wmi_bb_timing_cfg_arg *arg) { struct wmi_pdev_bb_timing_cfg_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_bb_timing_cfg_cmd *)skb->data; cmd->bb_tx_timing = __cpu_to_le32(arg->bb_tx_timing); cmd->bb_xpa_timing = __cpu_to_le32(arg->bb_xpa_timing); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev bb_tx_timing 0x%x bb_xpa_timing 0x%x\n", arg->bb_tx_timing, arg->bb_xpa_timing); return skb; } static const struct wmi_ops wmi_ops = { .rx = ath10k_wmi_op_rx, .map_svc = wmi_main_svc_map, .pull_scan = ath10k_wmi_op_pull_scan_ev, .pull_mgmt_rx = ath10k_wmi_op_pull_mgmt_rx_ev, .pull_ch_info = ath10k_wmi_op_pull_ch_info_ev, .pull_vdev_start = ath10k_wmi_op_pull_vdev_start_ev, .pull_peer_kick = ath10k_wmi_op_pull_peer_kick_ev, .pull_swba = ath10k_wmi_op_pull_swba_ev, .pull_phyerr_hdr = ath10k_wmi_op_pull_phyerr_ev_hdr, .pull_phyerr = ath10k_wmi_op_pull_phyerr_ev, .pull_svc_rdy = ath10k_wmi_main_op_pull_svc_rdy_ev, .pull_rdy = ath10k_wmi_op_pull_rdy_ev, .pull_fw_stats = ath10k_wmi_main_op_pull_fw_stats, .pull_roam_ev = ath10k_wmi_op_pull_roam_ev, .pull_echo_ev = ath10k_wmi_op_pull_echo_ev, .gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend, .gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume, .gen_pdev_set_rd = ath10k_wmi_op_gen_pdev_set_rd, .gen_pdev_set_param = ath10k_wmi_op_gen_pdev_set_param, .gen_init = ath10k_wmi_op_gen_init, .gen_start_scan = ath10k_wmi_op_gen_start_scan, .gen_stop_scan = ath10k_wmi_op_gen_stop_scan, .gen_vdev_create = ath10k_wmi_op_gen_vdev_create, .gen_vdev_delete = ath10k_wmi_op_gen_vdev_delete, .gen_vdev_start = ath10k_wmi_op_gen_vdev_start, .gen_vdev_stop = ath10k_wmi_op_gen_vdev_stop, .gen_vdev_up = ath10k_wmi_op_gen_vdev_up, .gen_vdev_down = ath10k_wmi_op_gen_vdev_down, .gen_vdev_set_param = ath10k_wmi_op_gen_vdev_set_param, .gen_vdev_install_key = ath10k_wmi_op_gen_vdev_install_key, .gen_vdev_spectral_conf = ath10k_wmi_op_gen_vdev_spectral_conf, .gen_vdev_spectral_enable = ath10k_wmi_op_gen_vdev_spectral_enable, /* .gen_vdev_wmm_conf not implemented */ .gen_peer_create = ath10k_wmi_op_gen_peer_create, .gen_peer_delete = ath10k_wmi_op_gen_peer_delete, .gen_peer_flush = ath10k_wmi_op_gen_peer_flush, .gen_peer_set_param = ath10k_wmi_op_gen_peer_set_param, .gen_peer_assoc = ath10k_wmi_op_gen_peer_assoc, .gen_set_psmode = ath10k_wmi_op_gen_set_psmode, .gen_set_sta_ps = ath10k_wmi_op_gen_set_sta_ps, .gen_set_ap_ps = ath10k_wmi_op_gen_set_ap_ps, .gen_scan_chan_list = ath10k_wmi_op_gen_scan_chan_list, .gen_beacon_dma = ath10k_wmi_op_gen_beacon_dma, .gen_pdev_set_wmm = ath10k_wmi_op_gen_pdev_set_wmm, .gen_request_stats = ath10k_wmi_op_gen_request_stats, .gen_force_fw_hang = ath10k_wmi_op_gen_force_fw_hang, .gen_mgmt_tx = ath10k_wmi_op_gen_mgmt_tx, .gen_dbglog_cfg = ath10k_wmi_op_gen_dbglog_cfg, .gen_pktlog_enable = ath10k_wmi_op_gen_pktlog_enable, .gen_pktlog_disable = ath10k_wmi_op_gen_pktlog_disable, .gen_pdev_set_quiet_mode = ath10k_wmi_op_gen_pdev_set_quiet_mode, /* .gen_pdev_get_temperature not implemented */ .gen_addba_clear_resp = ath10k_wmi_op_gen_addba_clear_resp, .gen_addba_send = ath10k_wmi_op_gen_addba_send, .gen_addba_set_resp = ath10k_wmi_op_gen_addba_set_resp, .gen_delba_send = ath10k_wmi_op_gen_delba_send, .fw_stats_fill = ath10k_wmi_main_op_fw_stats_fill, .get_vdev_subtype = ath10k_wmi_op_get_vdev_subtype, .gen_echo = ath10k_wmi_op_gen_echo, .gen_gpio_config = ath10k_wmi_op_gen_gpio_config, .gen_gpio_output = ath10k_wmi_op_gen_gpio_output, /* .gen_bcn_tmpl not implemented */ /* .gen_prb_tmpl not implemented */ /* .gen_p2p_go_bcn_ie not implemented */ /* .gen_adaptive_qcs not implemented */ /* .gen_pdev_enable_adaptive_cca not implemented */ }; static const struct wmi_ops wmi_10_1_ops = { .rx = ath10k_wmi_10_1_op_rx, .map_svc = wmi_10x_svc_map, .pull_svc_rdy = ath10k_wmi_10x_op_pull_svc_rdy_ev, .pull_fw_stats = ath10k_wmi_10x_op_pull_fw_stats, .gen_init = ath10k_wmi_10_1_op_gen_init, .gen_pdev_set_rd = ath10k_wmi_10x_op_gen_pdev_set_rd, .gen_start_scan = ath10k_wmi_10x_op_gen_start_scan, .gen_peer_assoc = ath10k_wmi_10_1_op_gen_peer_assoc, /* .gen_pdev_get_temperature not implemented */ /* shared with main branch */ .pull_scan = ath10k_wmi_op_pull_scan_ev, .pull_mgmt_rx = ath10k_wmi_op_pull_mgmt_rx_ev, .pull_ch_info = ath10k_wmi_op_pull_ch_info_ev, .pull_vdev_start = ath10k_wmi_op_pull_vdev_start_ev, .pull_peer_kick = ath10k_wmi_op_pull_peer_kick_ev, .pull_swba = ath10k_wmi_op_pull_swba_ev, .pull_phyerr_hdr = ath10k_wmi_op_pull_phyerr_ev_hdr, .pull_phyerr = ath10k_wmi_op_pull_phyerr_ev, .pull_rdy = ath10k_wmi_op_pull_rdy_ev, .pull_roam_ev = ath10k_wmi_op_pull_roam_ev, .pull_echo_ev = ath10k_wmi_op_pull_echo_ev, .gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend, .gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume, .gen_pdev_set_param = ath10k_wmi_op_gen_pdev_set_param, .gen_stop_scan = ath10k_wmi_op_gen_stop_scan, .gen_vdev_create = ath10k_wmi_op_gen_vdev_create, .gen_vdev_delete = ath10k_wmi_op_gen_vdev_delete, .gen_vdev_start = ath10k_wmi_op_gen_vdev_start, .gen_vdev_stop = ath10k_wmi_op_gen_vdev_stop, .gen_vdev_up = ath10k_wmi_op_gen_vdev_up, .gen_vdev_down = ath10k_wmi_op_gen_vdev_down, .gen_vdev_set_param = ath10k_wmi_op_gen_vdev_set_param, .gen_vdev_install_key = ath10k_wmi_op_gen_vdev_install_key, .gen_vdev_spectral_conf = ath10k_wmi_op_gen_vdev_spectral_conf, .gen_vdev_spectral_enable = ath10k_wmi_op_gen_vdev_spectral_enable, /* .gen_vdev_wmm_conf not implemented */ .gen_peer_create = ath10k_wmi_op_gen_peer_create, .gen_peer_delete = ath10k_wmi_op_gen_peer_delete, .gen_peer_flush = ath10k_wmi_op_gen_peer_flush, .gen_peer_set_param = ath10k_wmi_op_gen_peer_set_param, .gen_set_psmode = ath10k_wmi_op_gen_set_psmode, .gen_set_sta_ps = ath10k_wmi_op_gen_set_sta_ps, .gen_set_ap_ps = ath10k_wmi_op_gen_set_ap_ps, .gen_scan_chan_list = ath10k_wmi_op_gen_scan_chan_list, .gen_beacon_dma = ath10k_wmi_op_gen_beacon_dma, .gen_pdev_set_wmm = ath10k_wmi_op_gen_pdev_set_wmm, .gen_request_stats = ath10k_wmi_op_gen_request_stats, .gen_force_fw_hang = ath10k_wmi_op_gen_force_fw_hang, .gen_mgmt_tx = ath10k_wmi_op_gen_mgmt_tx, .gen_dbglog_cfg = ath10k_wmi_op_gen_dbglog_cfg, .gen_pktlog_enable = ath10k_wmi_op_gen_pktlog_enable, .gen_pktlog_disable = ath10k_wmi_op_gen_pktlog_disable, .gen_pdev_set_quiet_mode = ath10k_wmi_op_gen_pdev_set_quiet_mode, .gen_addba_clear_resp = ath10k_wmi_op_gen_addba_clear_resp, .gen_addba_send = ath10k_wmi_op_gen_addba_send, .gen_addba_set_resp = ath10k_wmi_op_gen_addba_set_resp, .gen_delba_send = ath10k_wmi_op_gen_delba_send, .fw_stats_fill = ath10k_wmi_10x_op_fw_stats_fill, .get_vdev_subtype = ath10k_wmi_op_get_vdev_subtype, .gen_echo = ath10k_wmi_op_gen_echo, .gen_gpio_config = ath10k_wmi_op_gen_gpio_config, .gen_gpio_output = ath10k_wmi_op_gen_gpio_output, /* .gen_bcn_tmpl not implemented */ /* .gen_prb_tmpl not implemented */ /* .gen_p2p_go_bcn_ie not implemented */ /* .gen_adaptive_qcs not implemented */ /* .gen_pdev_enable_adaptive_cca not implemented */ }; static const struct wmi_ops wmi_10_2_ops = { .rx = ath10k_wmi_10_2_op_rx, .pull_fw_stats = ath10k_wmi_10_2_op_pull_fw_stats, .gen_init = ath10k_wmi_10_2_op_gen_init, .gen_peer_assoc = ath10k_wmi_10_2_op_gen_peer_assoc, /* .gen_pdev_get_temperature not implemented */ /* shared with 10.1 */ .map_svc = wmi_10x_svc_map, .pull_svc_rdy = ath10k_wmi_10x_op_pull_svc_rdy_ev, .gen_pdev_set_rd = ath10k_wmi_10x_op_gen_pdev_set_rd, .gen_start_scan = ath10k_wmi_10x_op_gen_start_scan, .gen_echo = ath10k_wmi_op_gen_echo, .pull_scan = ath10k_wmi_op_pull_scan_ev, .pull_mgmt_rx = ath10k_wmi_op_pull_mgmt_rx_ev, .pull_ch_info = ath10k_wmi_op_pull_ch_info_ev, .pull_vdev_start = ath10k_wmi_op_pull_vdev_start_ev, .pull_peer_kick = ath10k_wmi_op_pull_peer_kick_ev, .pull_swba = ath10k_wmi_op_pull_swba_ev, .pull_phyerr_hdr = ath10k_wmi_op_pull_phyerr_ev_hdr, .pull_phyerr = ath10k_wmi_op_pull_phyerr_ev, .pull_rdy = ath10k_wmi_op_pull_rdy_ev, .pull_roam_ev = ath10k_wmi_op_pull_roam_ev, .pull_echo_ev = ath10k_wmi_op_pull_echo_ev, .gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend, .gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume, .gen_pdev_set_param = ath10k_wmi_op_gen_pdev_set_param, .gen_stop_scan = ath10k_wmi_op_gen_stop_scan, .gen_vdev_create = ath10k_wmi_op_gen_vdev_create, .gen_vdev_delete = ath10k_wmi_op_gen_vdev_delete, .gen_vdev_start = ath10k_wmi_op_gen_vdev_start, .gen_vdev_stop = ath10k_wmi_op_gen_vdev_stop, .gen_vdev_up = ath10k_wmi_op_gen_vdev_up, .gen_vdev_down = ath10k_wmi_op_gen_vdev_down, .gen_vdev_set_param = ath10k_wmi_op_gen_vdev_set_param, .gen_vdev_install_key = ath10k_wmi_op_gen_vdev_install_key, .gen_vdev_spectral_conf = ath10k_wmi_op_gen_vdev_spectral_conf, .gen_vdev_spectral_enable = ath10k_wmi_op_gen_vdev_spectral_enable, /* .gen_vdev_wmm_conf not implemented */ .gen_peer_create = ath10k_wmi_op_gen_peer_create, .gen_peer_delete = ath10k_wmi_op_gen_peer_delete, .gen_peer_flush = ath10k_wmi_op_gen_peer_flush, .gen_pdev_set_base_macaddr = ath10k_wmi_op_gen_pdev_set_base_macaddr, .gen_peer_set_param = ath10k_wmi_op_gen_peer_set_param, .gen_set_psmode = ath10k_wmi_op_gen_set_psmode, .gen_set_sta_ps = ath10k_wmi_op_gen_set_sta_ps, .gen_set_ap_ps = ath10k_wmi_op_gen_set_ap_ps, .gen_scan_chan_list = ath10k_wmi_op_gen_scan_chan_list, .gen_beacon_dma = ath10k_wmi_op_gen_beacon_dma, .gen_pdev_set_wmm = ath10k_wmi_op_gen_pdev_set_wmm, .gen_request_stats = ath10k_wmi_op_gen_request_stats, .gen_force_fw_hang = ath10k_wmi_op_gen_force_fw_hang, .gen_mgmt_tx = ath10k_wmi_op_gen_mgmt_tx, .gen_dbglog_cfg = ath10k_wmi_op_gen_dbglog_cfg, .gen_pktlog_enable = ath10k_wmi_op_gen_pktlog_enable, .gen_pktlog_disable = ath10k_wmi_op_gen_pktlog_disable, .gen_pdev_set_quiet_mode = ath10k_wmi_op_gen_pdev_set_quiet_mode, .gen_addba_clear_resp = ath10k_wmi_op_gen_addba_clear_resp, .gen_addba_send = ath10k_wmi_op_gen_addba_send, .gen_addba_set_resp = ath10k_wmi_op_gen_addba_set_resp, .gen_delba_send = ath10k_wmi_op_gen_delba_send, .fw_stats_fill = ath10k_wmi_10x_op_fw_stats_fill, .get_vdev_subtype = ath10k_wmi_op_get_vdev_subtype, .gen_gpio_config = ath10k_wmi_op_gen_gpio_config, .gen_gpio_output = ath10k_wmi_op_gen_gpio_output, /* .gen_pdev_enable_adaptive_cca not implemented */ }; static const struct wmi_ops wmi_10_2_4_ops = { .rx = ath10k_wmi_10_2_op_rx, .pull_fw_stats = ath10k_wmi_10_2_4_op_pull_fw_stats, .gen_init = ath10k_wmi_10_2_op_gen_init, .gen_peer_assoc = ath10k_wmi_10_2_op_gen_peer_assoc, .gen_pdev_get_temperature = ath10k_wmi_10_2_op_gen_pdev_get_temperature, .gen_pdev_bss_chan_info_req = ath10k_wmi_10_2_op_gen_pdev_bss_chan_info, /* shared with 10.1 */ .map_svc = wmi_10x_svc_map, .pull_svc_rdy = ath10k_wmi_10x_op_pull_svc_rdy_ev, .gen_pdev_set_rd = ath10k_wmi_10x_op_gen_pdev_set_rd, .gen_start_scan = ath10k_wmi_10x_op_gen_start_scan, .gen_echo = ath10k_wmi_op_gen_echo, .pull_scan = ath10k_wmi_op_pull_scan_ev, .pull_mgmt_rx = ath10k_wmi_op_pull_mgmt_rx_ev, .pull_ch_info = ath10k_wmi_op_pull_ch_info_ev, .pull_vdev_start = ath10k_wmi_op_pull_vdev_start_ev, .pull_peer_kick = ath10k_wmi_op_pull_peer_kick_ev, .pull_swba = ath10k_wmi_10_2_4_op_pull_swba_ev, .pull_phyerr_hdr = ath10k_wmi_op_pull_phyerr_ev_hdr, .pull_phyerr = ath10k_wmi_op_pull_phyerr_ev, .pull_rdy = ath10k_wmi_op_pull_rdy_ev, .pull_roam_ev = ath10k_wmi_op_pull_roam_ev, .pull_echo_ev = ath10k_wmi_op_pull_echo_ev, .gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend, .gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume, .gen_pdev_set_param = ath10k_wmi_op_gen_pdev_set_param, .gen_stop_scan = ath10k_wmi_op_gen_stop_scan, .gen_vdev_create = ath10k_wmi_op_gen_vdev_create, .gen_vdev_delete = ath10k_wmi_op_gen_vdev_delete, .gen_vdev_start = ath10k_wmi_op_gen_vdev_start, .gen_vdev_stop = ath10k_wmi_op_gen_vdev_stop, .gen_vdev_up = ath10k_wmi_op_gen_vdev_up, .gen_vdev_down = ath10k_wmi_op_gen_vdev_down, .gen_vdev_set_param = ath10k_wmi_op_gen_vdev_set_param, .gen_vdev_install_key = ath10k_wmi_op_gen_vdev_install_key, .gen_vdev_spectral_conf = ath10k_wmi_op_gen_vdev_spectral_conf, .gen_vdev_spectral_enable = ath10k_wmi_op_gen_vdev_spectral_enable, .gen_peer_create = ath10k_wmi_op_gen_peer_create, .gen_peer_delete = ath10k_wmi_op_gen_peer_delete, .gen_peer_flush = ath10k_wmi_op_gen_peer_flush, .gen_peer_set_param = ath10k_wmi_op_gen_peer_set_param, .gen_set_psmode = ath10k_wmi_op_gen_set_psmode, .gen_set_sta_ps = ath10k_wmi_op_gen_set_sta_ps, .gen_set_ap_ps = ath10k_wmi_op_gen_set_ap_ps, .gen_scan_chan_list = ath10k_wmi_op_gen_scan_chan_list, .gen_beacon_dma = ath10k_wmi_op_gen_beacon_dma, .gen_pdev_set_wmm = ath10k_wmi_op_gen_pdev_set_wmm, .gen_request_stats = ath10k_wmi_op_gen_request_stats, .gen_force_fw_hang = ath10k_wmi_op_gen_force_fw_hang, .gen_mgmt_tx = ath10k_wmi_op_gen_mgmt_tx, .gen_dbglog_cfg = ath10k_wmi_op_gen_dbglog_cfg, .gen_pktlog_enable = ath10k_wmi_op_gen_pktlog_enable, .gen_pktlog_disable = ath10k_wmi_op_gen_pktlog_disable, .gen_pdev_set_quiet_mode = ath10k_wmi_op_gen_pdev_set_quiet_mode, .gen_addba_clear_resp = ath10k_wmi_op_gen_addba_clear_resp, .gen_addba_send = ath10k_wmi_op_gen_addba_send, .gen_addba_set_resp = ath10k_wmi_op_gen_addba_set_resp, .gen_delba_send = ath10k_wmi_op_gen_delba_send, .gen_pdev_get_tpc_config = ath10k_wmi_10_2_4_op_gen_pdev_get_tpc_config, .fw_stats_fill = ath10k_wmi_10x_op_fw_stats_fill, .gen_pdev_enable_adaptive_cca = ath10k_wmi_op_gen_pdev_enable_adaptive_cca, .get_vdev_subtype = ath10k_wmi_10_2_4_op_get_vdev_subtype, .gen_bb_timing = ath10k_wmi_10_2_4_op_gen_bb_timing, .gen_gpio_config = ath10k_wmi_op_gen_gpio_config, .gen_gpio_output = ath10k_wmi_op_gen_gpio_output, /* .gen_bcn_tmpl not implemented */ /* .gen_prb_tmpl not implemented */ /* .gen_p2p_go_bcn_ie not implemented */ /* .gen_adaptive_qcs not implemented */ }; static const struct wmi_ops wmi_10_4_ops = { .rx = ath10k_wmi_10_4_op_rx, .map_svc = wmi_10_4_svc_map, .pull_fw_stats = ath10k_wmi_10_4_op_pull_fw_stats, .pull_scan = ath10k_wmi_op_pull_scan_ev, .pull_mgmt_rx = ath10k_wmi_10_4_op_pull_mgmt_rx_ev, .pull_ch_info = ath10k_wmi_10_4_op_pull_ch_info_ev, .pull_vdev_start = ath10k_wmi_op_pull_vdev_start_ev, .pull_peer_kick = ath10k_wmi_op_pull_peer_kick_ev, .pull_swba = ath10k_wmi_10_4_op_pull_swba_ev, .pull_phyerr_hdr = ath10k_wmi_10_4_op_pull_phyerr_ev_hdr, .pull_phyerr = ath10k_wmi_10_4_op_pull_phyerr_ev, .pull_svc_rdy = ath10k_wmi_main_op_pull_svc_rdy_ev, .pull_rdy = ath10k_wmi_op_pull_rdy_ev, .pull_roam_ev = ath10k_wmi_op_pull_roam_ev, .pull_dfs_status_ev = ath10k_wmi_10_4_op_pull_dfs_status_ev, .get_txbf_conf_scheme = ath10k_wmi_10_4_txbf_conf_scheme, .gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend, .gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume, .gen_pdev_set_base_macaddr = ath10k_wmi_op_gen_pdev_set_base_macaddr, .gen_pdev_set_rd = ath10k_wmi_10x_op_gen_pdev_set_rd, .gen_pdev_set_param = ath10k_wmi_op_gen_pdev_set_param, .gen_init = ath10k_wmi_10_4_op_gen_init, .gen_start_scan = ath10k_wmi_op_gen_start_scan, .gen_stop_scan = ath10k_wmi_op_gen_stop_scan, .gen_vdev_create = ath10k_wmi_op_gen_vdev_create, .gen_vdev_delete = ath10k_wmi_op_gen_vdev_delete, .gen_vdev_start = ath10k_wmi_op_gen_vdev_start, .gen_vdev_stop = ath10k_wmi_op_gen_vdev_stop, .gen_vdev_up = ath10k_wmi_op_gen_vdev_up, .gen_vdev_down = ath10k_wmi_op_gen_vdev_down, .gen_vdev_set_param = ath10k_wmi_op_gen_vdev_set_param, .gen_vdev_install_key = ath10k_wmi_op_gen_vdev_install_key, .gen_vdev_spectral_conf = ath10k_wmi_op_gen_vdev_spectral_conf, .gen_vdev_spectral_enable = ath10k_wmi_op_gen_vdev_spectral_enable, .gen_peer_create = ath10k_wmi_op_gen_peer_create, .gen_peer_delete = ath10k_wmi_op_gen_peer_delete, .gen_peer_flush = ath10k_wmi_op_gen_peer_flush, .gen_peer_set_param = ath10k_wmi_op_gen_peer_set_param, .gen_peer_assoc = ath10k_wmi_10_4_op_gen_peer_assoc, .gen_set_psmode = ath10k_wmi_op_gen_set_psmode, .gen_set_sta_ps = ath10k_wmi_op_gen_set_sta_ps, .gen_set_ap_ps = ath10k_wmi_op_gen_set_ap_ps, .gen_scan_chan_list = ath10k_wmi_op_gen_scan_chan_list, .gen_beacon_dma = ath10k_wmi_op_gen_beacon_dma, .gen_pdev_set_wmm = ath10k_wmi_op_gen_pdev_set_wmm, .gen_force_fw_hang = ath10k_wmi_op_gen_force_fw_hang, .gen_mgmt_tx = ath10k_wmi_op_gen_mgmt_tx, .gen_dbglog_cfg = ath10k_wmi_10_4_op_gen_dbglog_cfg, .gen_pktlog_enable = ath10k_wmi_op_gen_pktlog_enable, .gen_pktlog_disable = ath10k_wmi_op_gen_pktlog_disable, .gen_pdev_set_quiet_mode = ath10k_wmi_op_gen_pdev_set_quiet_mode, .gen_addba_clear_resp = ath10k_wmi_op_gen_addba_clear_resp, .gen_addba_send = ath10k_wmi_op_gen_addba_send, .gen_addba_set_resp = ath10k_wmi_op_gen_addba_set_resp, .gen_delba_send = ath10k_wmi_op_gen_delba_send, .fw_stats_fill = ath10k_wmi_10_4_op_fw_stats_fill, .ext_resource_config = ath10k_wmi_10_4_ext_resource_config, .gen_update_fw_tdls_state = ath10k_wmi_10_4_gen_update_fw_tdls_state, .gen_tdls_peer_update = ath10k_wmi_10_4_gen_tdls_peer_update, .gen_pdev_get_tpc_table_cmdid = ath10k_wmi_10_4_op_gen_pdev_get_tpc_table_cmdid, .gen_radar_found = ath10k_wmi_10_4_gen_radar_found, .gen_per_peer_per_tid_cfg = ath10k_wmi_10_4_gen_per_peer_per_tid_cfg, /* shared with 10.2 */ .pull_echo_ev = ath10k_wmi_op_pull_echo_ev, .gen_request_stats = ath10k_wmi_op_gen_request_stats, .gen_pdev_get_temperature = ath10k_wmi_10_2_op_gen_pdev_get_temperature, .get_vdev_subtype = ath10k_wmi_10_4_op_get_vdev_subtype, .gen_pdev_bss_chan_info_req = ath10k_wmi_10_2_op_gen_pdev_bss_chan_info, .gen_echo = ath10k_wmi_op_gen_echo, .gen_pdev_get_tpc_config = ath10k_wmi_10_2_4_op_gen_pdev_get_tpc_config, .gen_gpio_config = ath10k_wmi_op_gen_gpio_config, .gen_gpio_output = ath10k_wmi_op_gen_gpio_output, }; int ath10k_wmi_attach(struct ath10k *ar) { switch (ar->running_fw->fw_file.wmi_op_version) { case ATH10K_FW_WMI_OP_VERSION_10_4: ar->wmi.ops = &wmi_10_4_ops; ar->wmi.cmd = &wmi_10_4_cmd_map; ar->wmi.vdev_param = &wmi_10_4_vdev_param_map; ar->wmi.pdev_param = &wmi_10_4_pdev_param_map; ar->wmi.peer_param = &wmi_peer_param_map; ar->wmi.peer_flags = &wmi_10_2_peer_flags_map; ar->wmi_key_cipher = wmi_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_10_2_4: ar->wmi.cmd = &wmi_10_2_4_cmd_map; ar->wmi.ops = &wmi_10_2_4_ops; ar->wmi.vdev_param = &wmi_10_2_4_vdev_param_map; ar->wmi.pdev_param = &wmi_10_2_4_pdev_param_map; ar->wmi.peer_param = &wmi_peer_param_map; ar->wmi.peer_flags = &wmi_10_2_peer_flags_map; ar->wmi_key_cipher = wmi_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_10_2: ar->wmi.cmd = &wmi_10_2_cmd_map; ar->wmi.ops = &wmi_10_2_ops; ar->wmi.vdev_param = &wmi_10x_vdev_param_map; ar->wmi.pdev_param = &wmi_10x_pdev_param_map; ar->wmi.peer_param = &wmi_peer_param_map; ar->wmi.peer_flags = &wmi_10_2_peer_flags_map; ar->wmi_key_cipher = wmi_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_10_1: ar->wmi.cmd = &wmi_10x_cmd_map; ar->wmi.ops = &wmi_10_1_ops; ar->wmi.vdev_param = &wmi_10x_vdev_param_map; ar->wmi.pdev_param = &wmi_10x_pdev_param_map; ar->wmi.peer_param = &wmi_peer_param_map; ar->wmi.peer_flags = &wmi_10x_peer_flags_map; ar->wmi_key_cipher = wmi_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_MAIN: ar->wmi.cmd = &wmi_cmd_map; ar->wmi.ops = &wmi_ops; ar->wmi.vdev_param = &wmi_vdev_param_map; ar->wmi.pdev_param = &wmi_pdev_param_map; ar->wmi.peer_param = &wmi_peer_param_map; ar->wmi.peer_flags = &wmi_peer_flags_map; ar->wmi_key_cipher = wmi_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_TLV: ath10k_wmi_tlv_attach(ar); ar->wmi_key_cipher = wmi_tlv_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_UNSET: case ATH10K_FW_WMI_OP_VERSION_MAX: ath10k_err(ar, "unsupported WMI op version: %d\n", ar->running_fw->fw_file.wmi_op_version); return -EINVAL; } init_completion(&ar->wmi.service_ready); init_completion(&ar->wmi.unified_ready); init_completion(&ar->wmi.barrier); init_completion(&ar->wmi.radar_confirm); INIT_WORK(&ar->svc_rdy_work, ath10k_wmi_event_service_ready_work); INIT_WORK(&ar->radar_confirmation_work, ath10k_radar_confirmation_work); if (test_bit(ATH10K_FW_FEATURE_MGMT_TX_BY_REF, ar->running_fw->fw_file.fw_features)) { idr_init(&ar->wmi.mgmt_pending_tx); } return 0; } void ath10k_wmi_free_host_mem(struct ath10k *ar) { int i; /* free the host memory chunks requested by firmware */ for (i = 0; i < ar->wmi.num_mem_chunks; i++) { dma_free_coherent(ar->dev, ar->wmi.mem_chunks[i].len, ar->wmi.mem_chunks[i].vaddr, ar->wmi.mem_chunks[i].paddr); } ar->wmi.num_mem_chunks = 0; } static int ath10k_wmi_mgmt_tx_clean_up_pending(int msdu_id, void *ptr, void *ctx) { struct ath10k_mgmt_tx_pkt_addr *pkt_addr = ptr; struct ath10k *ar = ctx; struct sk_buff *msdu; ath10k_dbg(ar, ATH10K_DBG_WMI, "force cleanup mgmt msdu_id %u\n", msdu_id); msdu = pkt_addr->vaddr; dma_unmap_single(ar->dev, pkt_addr->paddr, msdu->len, DMA_TO_DEVICE); ieee80211_free_txskb(ar->hw, msdu); kfree(pkt_addr); return 0; } void ath10k_wmi_detach(struct ath10k *ar) { if (test_bit(ATH10K_FW_FEATURE_MGMT_TX_BY_REF, ar->running_fw->fw_file.fw_features)) { spin_lock_bh(&ar->data_lock); idr_for_each(&ar->wmi.mgmt_pending_tx, ath10k_wmi_mgmt_tx_clean_up_pending, ar); idr_destroy(&ar->wmi.mgmt_pending_tx); spin_unlock_bh(&ar->data_lock); } cancel_work_sync(&ar->svc_rdy_work); dev_kfree_skb(ar->svc_rdy_skb); } |
| 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 | // SPDX-License-Identifier: GPL-2.0-or-later /* CacheFiles path walking and related routines * * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */ #include <linux/fs.h> #include <linux/namei.h> #include "internal.h" /* * Mark the backing file as being a cache file if it's not already in use. The * mark tells the culling request command that it's not allowed to cull the * file or directory. The caller must hold the inode lock. */ static bool __cachefiles_mark_inode_in_use(struct cachefiles_object *object, struct inode *inode) { bool can_use = false; if (!(inode->i_flags & S_KERNEL_FILE)) { inode->i_flags |= S_KERNEL_FILE; trace_cachefiles_mark_active(object, inode); can_use = true; } else { trace_cachefiles_mark_failed(object, inode); } return can_use; } static bool cachefiles_mark_inode_in_use(struct cachefiles_object *object, struct inode *inode) { bool can_use; inode_lock(inode); can_use = __cachefiles_mark_inode_in_use(object, inode); inode_unlock(inode); return can_use; } /* * Unmark a backing inode. The caller must hold the inode lock. */ static void __cachefiles_unmark_inode_in_use(struct cachefiles_object *object, struct inode *inode) { inode->i_flags &= ~S_KERNEL_FILE; trace_cachefiles_mark_inactive(object, inode); } static void cachefiles_do_unmark_inode_in_use(struct cachefiles_object *object, struct inode *inode) { inode_lock(inode); __cachefiles_unmark_inode_in_use(object, inode); inode_unlock(inode); } /* * Unmark a backing inode and tell cachefilesd that there's something that can * be culled. */ void cachefiles_unmark_inode_in_use(struct cachefiles_object *object, struct file *file) { struct cachefiles_cache *cache = object->volume->cache; struct inode *inode = file_inode(file); cachefiles_do_unmark_inode_in_use(object, inode); if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) { atomic_long_add(inode->i_blocks, &cache->b_released); if (atomic_inc_return(&cache->f_released)) cachefiles_state_changed(cache); } } /* * get a subdirectory */ struct dentry *cachefiles_get_directory(struct cachefiles_cache *cache, struct dentry *dir, const char *dirname, bool *_is_new) { struct dentry *subdir; struct path path; int ret; _enter(",,%s", dirname); /* search the current directory for the element name */ inode_lock_nested(d_inode(dir), I_MUTEX_PARENT); retry: ret = cachefiles_inject_read_error(); if (ret == 0) subdir = lookup_one(&nop_mnt_idmap, &QSTR(dirname), dir); else subdir = ERR_PTR(ret); trace_cachefiles_lookup(NULL, dir, subdir); if (IS_ERR(subdir)) { trace_cachefiles_vfs_error(NULL, d_backing_inode(dir), PTR_ERR(subdir), cachefiles_trace_lookup_error); if (PTR_ERR(subdir) == -ENOMEM) goto nomem_d_alloc; goto lookup_error; } _debug("subdir -> %pd %s", subdir, d_backing_inode(subdir) ? "positive" : "negative"); /* we need to create the subdir if it doesn't exist yet */ if (d_is_negative(subdir)) { ret = cachefiles_has_space(cache, 1, 0, cachefiles_has_space_for_create); if (ret < 0) goto mkdir_error; _debug("attempt mkdir"); path.mnt = cache->mnt; path.dentry = dir; ret = security_path_mkdir(&path, subdir, 0700); if (ret < 0) goto mkdir_error; ret = cachefiles_inject_write_error(); if (ret == 0) subdir = vfs_mkdir(&nop_mnt_idmap, d_inode(dir), subdir, 0700); else subdir = ERR_PTR(ret); if (IS_ERR(subdir)) { trace_cachefiles_vfs_error(NULL, d_inode(dir), ret, cachefiles_trace_mkdir_error); goto mkdir_error; } trace_cachefiles_mkdir(dir, subdir); if (unlikely(d_unhashed(subdir) || d_is_negative(subdir))) { dput(subdir); goto retry; } ASSERT(d_backing_inode(subdir)); _debug("mkdir -> %pd{ino=%lu}", subdir, d_backing_inode(subdir)->i_ino); if (_is_new) *_is_new = true; } /* Tell rmdir() it's not allowed to delete the subdir */ inode_lock(d_inode(subdir)); inode_unlock(d_inode(dir)); if (!__cachefiles_mark_inode_in_use(NULL, d_inode(subdir))) { pr_notice("cachefiles: Inode already in use: %pd (B=%lx)\n", subdir, d_inode(subdir)->i_ino); goto mark_error; } inode_unlock(d_inode(subdir)); /* we need to make sure the subdir is a directory */ ASSERT(d_backing_inode(subdir)); if (!d_can_lookup(subdir)) { pr_err("%s is not a directory\n", dirname); ret = -EIO; goto check_error; } ret = -EPERM; if (!(d_backing_inode(subdir)->i_opflags & IOP_XATTR) || !d_backing_inode(subdir)->i_op->lookup || !d_backing_inode(subdir)->i_op->mkdir || !d_backing_inode(subdir)->i_op->rename || !d_backing_inode(subdir)->i_op->rmdir || !d_backing_inode(subdir)->i_op->unlink) goto check_error; _leave(" = [%lu]", d_backing_inode(subdir)->i_ino); return subdir; check_error: cachefiles_put_directory(subdir); _leave(" = %d [check]", ret); return ERR_PTR(ret); mark_error: inode_unlock(d_inode(subdir)); dput(subdir); return ERR_PTR(-EBUSY); mkdir_error: inode_unlock(d_inode(dir)); if (!IS_ERR(subdir)) dput(subdir); pr_err("mkdir %s failed with error %d\n", dirname, ret); return ERR_PTR(ret); lookup_error: inode_unlock(d_inode(dir)); ret = PTR_ERR(subdir); pr_err("Lookup %s failed with error %d\n", dirname, ret); return ERR_PTR(ret); nomem_d_alloc: inode_unlock(d_inode(dir)); _leave(" = -ENOMEM"); return ERR_PTR(-ENOMEM); } /* * Put a subdirectory. */ void cachefiles_put_directory(struct dentry *dir) { if (dir) { cachefiles_do_unmark_inode_in_use(NULL, d_inode(dir)); dput(dir); } } /* * Remove a regular file from the cache. */ static int cachefiles_unlink(struct cachefiles_cache *cache, struct cachefiles_object *object, struct dentry *dir, struct dentry *dentry, enum fscache_why_object_killed why) { struct path path = { .mnt = cache->mnt, .dentry = dir, }; int ret; trace_cachefiles_unlink(object, d_inode(dentry)->i_ino, why); ret = security_path_unlink(&path, dentry); if (ret < 0) { cachefiles_io_error(cache, "Unlink security error"); return ret; } ret = cachefiles_inject_remove_error(); if (ret == 0) { ret = vfs_unlink(&nop_mnt_idmap, d_backing_inode(dir), dentry, NULL); if (ret == -EIO) cachefiles_io_error(cache, "Unlink failed"); } if (ret != 0) trace_cachefiles_vfs_error(object, d_backing_inode(dir), ret, cachefiles_trace_unlink_error); return ret; } /* * Delete an object representation from the cache * - File backed objects are unlinked * - Directory backed objects are stuffed into the graveyard for userspace to * delete */ int cachefiles_bury_object(struct cachefiles_cache *cache, struct cachefiles_object *object, struct dentry *dir, struct dentry *rep, enum fscache_why_object_killed why) { struct dentry *grave, *trap; struct path path, path_to_graveyard; char nbuffer[8 + 8 + 1]; int ret; _enter(",'%pd','%pd'", dir, rep); if (rep->d_parent != dir) { inode_unlock(d_inode(dir)); _leave(" = -ESTALE"); return -ESTALE; } /* non-directories can just be unlinked */ if (!d_is_dir(rep)) { dget(rep); /* Stop the dentry being negated if it's only pinned * by a file struct. */ ret = cachefiles_unlink(cache, object, dir, rep, why); dput(rep); inode_unlock(d_inode(dir)); _leave(" = %d", ret); return ret; } /* directories have to be moved to the graveyard */ _debug("move stale object to graveyard"); inode_unlock(d_inode(dir)); try_again: /* first step is to make up a grave dentry in the graveyard */ sprintf(nbuffer, "%08x%08x", (uint32_t) ktime_get_real_seconds(), (uint32_t) atomic_inc_return(&cache->gravecounter)); /* do the multiway lock magic */ trap = lock_rename(cache->graveyard, dir); if (IS_ERR(trap)) return PTR_ERR(trap); /* do some checks before getting the grave dentry */ if (rep->d_parent != dir || IS_DEADDIR(d_inode(rep))) { /* the entry was probably culled when we dropped the parent dir * lock */ unlock_rename(cache->graveyard, dir); _leave(" = 0 [culled?]"); return 0; } if (!d_can_lookup(cache->graveyard)) { unlock_rename(cache->graveyard, dir); cachefiles_io_error(cache, "Graveyard no longer a directory"); return -EIO; } if (trap == rep) { unlock_rename(cache->graveyard, dir); cachefiles_io_error(cache, "May not make directory loop"); return -EIO; } if (d_mountpoint(rep)) { unlock_rename(cache->graveyard, dir); cachefiles_io_error(cache, "Mountpoint in cache"); return -EIO; } grave = lookup_one(&nop_mnt_idmap, &QSTR(nbuffer), cache->graveyard); if (IS_ERR(grave)) { unlock_rename(cache->graveyard, dir); trace_cachefiles_vfs_error(object, d_inode(cache->graveyard), PTR_ERR(grave), cachefiles_trace_lookup_error); if (PTR_ERR(grave) == -ENOMEM) { _leave(" = -ENOMEM"); return -ENOMEM; } cachefiles_io_error(cache, "Lookup error %ld", PTR_ERR(grave)); return -EIO; } if (d_is_positive(grave)) { unlock_rename(cache->graveyard, dir); dput(grave); grave = NULL; cond_resched(); goto try_again; } if (d_mountpoint(grave)) { unlock_rename(cache->graveyard, dir); dput(grave); cachefiles_io_error(cache, "Mountpoint in graveyard"); return -EIO; } /* target should not be an ancestor of source */ if (trap == grave) { unlock_rename(cache->graveyard, dir); dput(grave); cachefiles_io_error(cache, "May not make directory loop"); return -EIO; } /* attempt the rename */ path.mnt = cache->mnt; path.dentry = dir; path_to_graveyard.mnt = cache->mnt; path_to_graveyard.dentry = cache->graveyard; ret = security_path_rename(&path, rep, &path_to_graveyard, grave, 0); if (ret < 0) { cachefiles_io_error(cache, "Rename security error %d", ret); } else { struct renamedata rd = { .mnt_idmap = &nop_mnt_idmap, .old_parent = dir, .old_dentry = rep, .new_parent = cache->graveyard, .new_dentry = grave, }; trace_cachefiles_rename(object, d_inode(rep)->i_ino, why); ret = cachefiles_inject_read_error(); if (ret == 0) ret = vfs_rename(&rd); if (ret != 0) trace_cachefiles_vfs_error(object, d_inode(dir), ret, cachefiles_trace_rename_error); if (ret != 0 && ret != -ENOMEM) cachefiles_io_error(cache, "Rename failed with error %d", ret); } __cachefiles_unmark_inode_in_use(object, d_inode(rep)); unlock_rename(cache->graveyard, dir); dput(grave); _leave(" = 0"); return 0; } /* * Delete a cache file. */ int cachefiles_delete_object(struct cachefiles_object *object, enum fscache_why_object_killed why) { struct cachefiles_volume *volume = object->volume; struct dentry *dentry = object->file->f_path.dentry; struct dentry *fan = volume->fanout[(u8)object->cookie->key_hash]; int ret; _enter(",OBJ%x{%pD}", object->debug_id, object->file); /* Stop the dentry being negated if it's only pinned by a file struct. */ dget(dentry); inode_lock_nested(d_backing_inode(fan), I_MUTEX_PARENT); ret = cachefiles_unlink(volume->cache, object, fan, dentry, why); inode_unlock(d_backing_inode(fan)); dput(dentry); return ret; } /* * Create a temporary file and leave it unattached and un-xattr'd until the * time comes to discard the object from memory. */ struct file *cachefiles_create_tmpfile(struct cachefiles_object *object) { struct cachefiles_volume *volume = object->volume; struct cachefiles_cache *cache = volume->cache; const struct cred *saved_cred; struct dentry *fan = volume->fanout[(u8)object->cookie->key_hash]; struct file *file; const struct path parentpath = { .mnt = cache->mnt, .dentry = fan }; uint64_t ni_size; long ret; cachefiles_begin_secure(cache, &saved_cred); ret = cachefiles_inject_write_error(); if (ret == 0) { file = kernel_tmpfile_open(&nop_mnt_idmap, &parentpath, S_IFREG | 0600, O_RDWR | O_LARGEFILE | O_DIRECT, cache->cache_cred); ret = PTR_ERR_OR_ZERO(file); } if (ret) { trace_cachefiles_vfs_error(object, d_inode(fan), ret, cachefiles_trace_tmpfile_error); if (ret == -EIO) cachefiles_io_error_obj(object, "Failed to create tmpfile"); goto err; } trace_cachefiles_tmpfile(object, file_inode(file)); /* This is a newly created file with no other possible user */ if (!cachefiles_mark_inode_in_use(object, file_inode(file))) WARN_ON(1); ret = cachefiles_ondemand_init_object(object); if (ret < 0) goto err_unuse; ni_size = object->cookie->object_size; ni_size = round_up(ni_size, CACHEFILES_DIO_BLOCK_SIZE); if (ni_size > 0) { trace_cachefiles_trunc(object, file_inode(file), 0, ni_size, cachefiles_trunc_expand_tmpfile); ret = cachefiles_inject_write_error(); if (ret == 0) ret = vfs_truncate(&file->f_path, ni_size); if (ret < 0) { trace_cachefiles_vfs_error( object, file_inode(file), ret, cachefiles_trace_trunc_error); goto err_unuse; } } ret = -EINVAL; if (unlikely(!file->f_op->read_iter) || unlikely(!file->f_op->write_iter)) { fput(file); pr_notice("Cache does not support read_iter and write_iter\n"); goto err_unuse; } out: cachefiles_end_secure(cache, saved_cred); return file; err_unuse: cachefiles_do_unmark_inode_in_use(object, file_inode(file)); fput(file); err: file = ERR_PTR(ret); goto out; } /* * Create a new file. */ static bool cachefiles_create_file(struct cachefiles_object *object) { struct file *file; int ret; ret = cachefiles_has_space(object->volume->cache, 1, 0, cachefiles_has_space_for_create); if (ret < 0) return false; file = cachefiles_create_tmpfile(object); if (IS_ERR(file)) return false; set_bit(FSCACHE_COOKIE_NEEDS_UPDATE, &object->cookie->flags); set_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags); _debug("create -> %pD{ino=%lu}", file, file_inode(file)->i_ino); object->file = file; return true; } /* * Open an existing file, checking its attributes and replacing it if it is * stale. */ static bool cachefiles_open_file(struct cachefiles_object *object, struct dentry *dentry) { struct cachefiles_cache *cache = object->volume->cache; struct file *file; struct path path; int ret; _enter("%pd", dentry); if (!cachefiles_mark_inode_in_use(object, d_inode(dentry))) { pr_notice("cachefiles: Inode already in use: %pd (B=%lx)\n", dentry, d_inode(dentry)->i_ino); return false; } /* We need to open a file interface onto a data file now as we can't do * it on demand because writeback called from do_exit() sees * current->fs == NULL - which breaks d_path() called from ext4 open. */ path.mnt = cache->mnt; path.dentry = dentry; file = kernel_file_open(&path, O_RDWR | O_LARGEFILE | O_DIRECT, cache->cache_cred); if (IS_ERR(file)) { trace_cachefiles_vfs_error(object, d_backing_inode(dentry), PTR_ERR(file), cachefiles_trace_open_error); goto error; } if (unlikely(!file->f_op->read_iter) || unlikely(!file->f_op->write_iter)) { pr_notice("Cache does not support read_iter and write_iter\n"); goto error_fput; } _debug("file -> %pd positive", dentry); ret = cachefiles_ondemand_init_object(object); if (ret < 0) goto error_fput; ret = cachefiles_check_auxdata(object, file); if (ret < 0) goto check_failed; clear_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &object->cookie->flags); object->file = file; /* Always update the atime on an object we've just looked up (this is * used to keep track of culling, and atimes are only updated by read, * write and readdir but not lookup or open). */ touch_atime(&file->f_path); return true; check_failed: fscache_cookie_lookup_negative(object->cookie); cachefiles_unmark_inode_in_use(object, file); fput(file); if (ret == -ESTALE) return cachefiles_create_file(object); return false; error_fput: fput(file); error: cachefiles_do_unmark_inode_in_use(object, d_inode(dentry)); return false; } /* * walk from the parent object to the child object through the backing * filesystem, creating directories as we go */ bool cachefiles_look_up_object(struct cachefiles_object *object) { struct cachefiles_volume *volume = object->volume; struct dentry *dentry, *fan = volume->fanout[(u8)object->cookie->key_hash]; int ret; _enter("OBJ%x,%s,", object->debug_id, object->d_name); /* Look up path "cache/vol/fanout/file". */ ret = cachefiles_inject_read_error(); if (ret == 0) dentry = lookup_one_positive_unlocked(&nop_mnt_idmap, &QSTR(object->d_name), fan); else dentry = ERR_PTR(ret); trace_cachefiles_lookup(object, fan, dentry); if (IS_ERR(dentry)) { if (dentry == ERR_PTR(-ENOENT)) goto new_file; if (dentry == ERR_PTR(-EIO)) cachefiles_io_error_obj(object, "Lookup failed"); return false; } if (!d_is_reg(dentry)) { pr_err("%pd is not a file\n", dentry); inode_lock_nested(d_inode(fan), I_MUTEX_PARENT); ret = cachefiles_bury_object(volume->cache, object, fan, dentry, FSCACHE_OBJECT_IS_WEIRD); dput(dentry); if (ret < 0) return false; goto new_file; } ret = cachefiles_open_file(object, dentry); dput(dentry); if (!ret) return false; _leave(" = t [%lu]", file_inode(object->file)->i_ino); return true; new_file: fscache_cookie_lookup_negative(object->cookie); return cachefiles_create_file(object); } /* * Attempt to link a temporary file into its rightful place in the cache. */ bool cachefiles_commit_tmpfile(struct cachefiles_cache *cache, struct cachefiles_object *object) { struct cachefiles_volume *volume = object->volume; struct dentry *dentry, *fan = volume->fanout[(u8)object->cookie->key_hash]; bool success = false; int ret; _enter(",%pD", object->file); inode_lock_nested(d_inode(fan), I_MUTEX_PARENT); ret = cachefiles_inject_read_error(); if (ret == 0) dentry = lookup_one(&nop_mnt_idmap, &QSTR(object->d_name), fan); else dentry = ERR_PTR(ret); if (IS_ERR(dentry)) { trace_cachefiles_vfs_error(object, d_inode(fan), PTR_ERR(dentry), cachefiles_trace_lookup_error); _debug("lookup fail %ld", PTR_ERR(dentry)); goto out_unlock; } if (!d_is_negative(dentry)) { ret = cachefiles_unlink(volume->cache, object, fan, dentry, FSCACHE_OBJECT_IS_STALE); if (ret < 0) goto out_dput; dput(dentry); ret = cachefiles_inject_read_error(); if (ret == 0) dentry = lookup_one(&nop_mnt_idmap, &QSTR(object->d_name), fan); else dentry = ERR_PTR(ret); if (IS_ERR(dentry)) { trace_cachefiles_vfs_error(object, d_inode(fan), PTR_ERR(dentry), cachefiles_trace_lookup_error); _debug("lookup fail %ld", PTR_ERR(dentry)); goto out_unlock; } } ret = cachefiles_inject_read_error(); if (ret == 0) ret = vfs_link(object->file->f_path.dentry, &nop_mnt_idmap, d_inode(fan), dentry, NULL); if (ret < 0) { trace_cachefiles_vfs_error(object, d_inode(fan), ret, cachefiles_trace_link_error); _debug("link fail %d", ret); } else { trace_cachefiles_link(object, file_inode(object->file)); spin_lock(&object->lock); /* TODO: Do we want to switch the file pointer to the new dentry? */ clear_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags); spin_unlock(&object->lock); success = true; } out_dput: dput(dentry); out_unlock: inode_unlock(d_inode(fan)); _leave(" = %u", success); return success; } /* * Look up an inode to be checked or culled. Return -EBUSY if the inode is * marked in use. */ static struct dentry *cachefiles_lookup_for_cull(struct cachefiles_cache *cache, struct dentry *dir, char *filename) { struct dentry *victim; int ret = -ENOENT; inode_lock_nested(d_inode(dir), I_MUTEX_PARENT); victim = lookup_one(&nop_mnt_idmap, &QSTR(filename), dir); if (IS_ERR(victim)) goto lookup_error; if (d_is_negative(victim)) goto lookup_put; if (d_inode(victim)->i_flags & S_KERNEL_FILE) goto lookup_busy; return victim; lookup_busy: ret = -EBUSY; lookup_put: inode_unlock(d_inode(dir)); dput(victim); return ERR_PTR(ret); lookup_error: inode_unlock(d_inode(dir)); ret = PTR_ERR(victim); if (ret == -ENOENT) return ERR_PTR(-ESTALE); /* Probably got retired by the netfs */ if (ret == -EIO) { cachefiles_io_error(cache, "Lookup failed"); } else if (ret != -ENOMEM) { pr_err("Internal error: %d\n", ret); ret = -EIO; } return ERR_PTR(ret); } /* * Cull an object if it's not in use * - called only by cache manager daemon */ int cachefiles_cull(struct cachefiles_cache *cache, struct dentry *dir, char *filename) { struct dentry *victim; struct inode *inode; int ret; _enter(",%pd/,%s", dir, filename); victim = cachefiles_lookup_for_cull(cache, dir, filename); if (IS_ERR(victim)) return PTR_ERR(victim); /* check to see if someone is using this object */ inode = d_inode(victim); inode_lock(inode); if (inode->i_flags & S_KERNEL_FILE) { ret = -EBUSY; } else { /* Stop the cache from picking it back up */ inode->i_flags |= S_KERNEL_FILE; ret = 0; } inode_unlock(inode); if (ret < 0) goto error_unlock; ret = cachefiles_bury_object(cache, NULL, dir, victim, FSCACHE_OBJECT_WAS_CULLED); if (ret < 0) goto error; fscache_count_culled(); dput(victim); _leave(" = 0"); return 0; error_unlock: inode_unlock(d_inode(dir)); error: dput(victim); if (ret == -ENOENT) return -ESTALE; /* Probably got retired by the netfs */ if (ret != -ENOMEM) { pr_err("Internal error: %d\n", ret); ret = -EIO; } _leave(" = %d", ret); return ret; } /* * Find out if an object is in use or not * - called only by cache manager daemon * - returns -EBUSY or 0 to indicate whether an object is in use or not */ int cachefiles_check_in_use(struct cachefiles_cache *cache, struct dentry *dir, char *filename) { struct dentry *victim; int ret = 0; victim = cachefiles_lookup_for_cull(cache, dir, filename); if (IS_ERR(victim)) return PTR_ERR(victim); inode_unlock(d_inode(dir)); dput(victim); return ret; } |
| 1 3 1 7 7 6 7 1 4 1 1 4 1 3 4 3 2 4 4 3 2 2 3 3 2 1 3 2 2 2 2 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 | // SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2003-2013 Jozsef Kadlecsik <kadlec@netfilter.org> */ /* Kernel module implementing an IP set type: the hash:ip,port type */ #include <linux/jhash.h> #include <linux/module.h> #include <linux/ip.h> #include <linux/skbuff.h> #include <linux/errno.h> #include <linux/random.h> #include <net/ip.h> #include <net/ipv6.h> #include <net/netlink.h> #include <net/tcp.h> #include <linux/netfilter.h> #include <linux/netfilter/ipset/pfxlen.h> #include <linux/netfilter/ipset/ip_set.h> #include <linux/netfilter/ipset/ip_set_getport.h> #include <linux/netfilter/ipset/ip_set_hash.h> #define IPSET_TYPE_REV_MIN 0 /* 1 SCTP and UDPLITE support added */ /* 2 Counters support added */ /* 3 Comments support added */ /* 4 Forceadd support added */ /* 5 skbinfo support added */ /* 6 bucketsize, initval support added */ #define IPSET_TYPE_REV_MAX 7 /* bitmask support added */ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Jozsef Kadlecsik <kadlec@netfilter.org>"); IP_SET_MODULE_DESC("hash:ip,port", IPSET_TYPE_REV_MIN, IPSET_TYPE_REV_MAX); MODULE_ALIAS("ip_set_hash:ip,port"); /* Type specific function prefix */ #define HTYPE hash_ipport #define IP_SET_HASH_WITH_NETMASK #define IP_SET_HASH_WITH_BITMASK /* IPv4 variant */ /* Member elements */ struct hash_ipport4_elem { __be32 ip; __be16 port; u8 proto; u8 padding; }; /* Common functions */ static bool hash_ipport4_data_equal(const struct hash_ipport4_elem *ip1, const struct hash_ipport4_elem *ip2, u32 *multi) { return ip1->ip == ip2->ip && ip1->port == ip2->port && ip1->proto == ip2->proto; } static bool hash_ipport4_data_list(struct sk_buff *skb, const struct hash_ipport4_elem *data) { if (nla_put_ipaddr4(skb, IPSET_ATTR_IP, data->ip) || nla_put_net16(skb, IPSET_ATTR_PORT, data->port) || nla_put_u8(skb, IPSET_ATTR_PROTO, data->proto)) goto nla_put_failure; return false; nla_put_failure: return true; } static void hash_ipport4_data_next(struct hash_ipport4_elem *next, const struct hash_ipport4_elem *d) { next->ip = d->ip; next->port = d->port; } #define MTYPE hash_ipport4 #define HOST_MASK 32 #include "ip_set_hash_gen.h" static int hash_ipport4_kadt(struct ip_set *set, const struct sk_buff *skb, const struct xt_action_param *par, enum ipset_adt adt, struct ip_set_adt_opt *opt) { ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_ipport4_elem e = { .ip = 0 }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); const struct MTYPE *h = set->data; if (!ip_set_get_ip4_port(skb, opt->flags & IPSET_DIM_TWO_SRC, &e.port, &e.proto)) return -EINVAL; ip4addrptr(skb, opt->flags & IPSET_DIM_ONE_SRC, &e.ip); e.ip &= h->bitmask.ip; if (e.ip == 0) return -EINVAL; return adtfn(set, &e, &ext, &opt->ext, opt->cmdflags); } static int hash_ipport4_uadt(struct ip_set *set, struct nlattr *tb[], enum ipset_adt adt, u32 *lineno, u32 flags, bool retried) { struct hash_ipport4 *h = set->data; ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_ipport4_elem e = { .ip = 0 }; struct ip_set_ext ext = IP_SET_INIT_UEXT(set); u32 ip, ip_to = 0, p = 0, port, port_to, i = 0; bool with_ports = false; int ret; if (tb[IPSET_ATTR_LINENO]) *lineno = nla_get_u32(tb[IPSET_ATTR_LINENO]); if (unlikely(!tb[IPSET_ATTR_IP] || !ip_set_attr_netorder(tb, IPSET_ATTR_PORT) || !ip_set_optattr_netorder(tb, IPSET_ATTR_PORT_TO))) return -IPSET_ERR_PROTOCOL; ret = ip_set_get_ipaddr4(tb[IPSET_ATTR_IP], &e.ip); if (ret) return ret; ret = ip_set_get_extensions(set, tb, &ext); if (ret) return ret; e.ip &= h->bitmask.ip; if (e.ip == 0) return -EINVAL; e.port = nla_get_be16(tb[IPSET_ATTR_PORT]); if (tb[IPSET_ATTR_PROTO]) { e.proto = nla_get_u8(tb[IPSET_ATTR_PROTO]); with_ports = ip_set_proto_with_ports(e.proto); if (e.proto == 0) return -IPSET_ERR_INVALID_PROTO; } else { return -IPSET_ERR_MISSING_PROTO; } if (!(with_ports || e.proto == IPPROTO_ICMP)) e.port = 0; if (adt == IPSET_TEST || !(tb[IPSET_ATTR_IP_TO] || tb[IPSET_ATTR_CIDR] || tb[IPSET_ATTR_PORT_TO])) { ret = adtfn(set, &e, &ext, &ext, flags); return ip_set_eexist(ret, flags) ? 0 : ret; } ip_to = ip = ntohl(e.ip); if (tb[IPSET_ATTR_IP_TO]) { ret = ip_set_get_hostipaddr4(tb[IPSET_ATTR_IP_TO], &ip_to); if (ret) return ret; if (ip > ip_to) swap(ip, ip_to); } else if (tb[IPSET_ATTR_CIDR]) { u8 cidr = nla_get_u8(tb[IPSET_ATTR_CIDR]); if (!cidr || cidr > HOST_MASK) return -IPSET_ERR_INVALID_CIDR; ip_set_mask_from_to(ip, ip_to, cidr); } port_to = port = ntohs(e.port); if (with_ports && tb[IPSET_ATTR_PORT_TO]) { port_to = ip_set_get_h16(tb[IPSET_ATTR_PORT_TO]); if (port > port_to) swap(port, port_to); } if (retried) ip = ntohl(h->next.ip); for (; ip <= ip_to; ip++) { p = retried && ip == ntohl(h->next.ip) ? ntohs(h->next.port) : port; for (; p <= port_to; p++, i++) { e.ip = htonl(ip); e.port = htons(p); if (i > IPSET_MAX_RANGE) { hash_ipport4_data_next(&h->next, &e); return -ERANGE; } ret = adtfn(set, &e, &ext, &ext, flags); if (ret && !ip_set_eexist(ret, flags)) return ret; ret = 0; } } return ret; } /* IPv6 variant */ struct hash_ipport6_elem { union nf_inet_addr ip; __be16 port; u8 proto; u8 padding; }; /* Common functions */ static bool hash_ipport6_data_equal(const struct hash_ipport6_elem *ip1, const struct hash_ipport6_elem *ip2, u32 *multi) { return ipv6_addr_equal(&ip1->ip.in6, &ip2->ip.in6) && ip1->port == ip2->port && ip1->proto == ip2->proto; } static bool hash_ipport6_data_list(struct sk_buff *skb, const struct hash_ipport6_elem *data) { if (nla_put_ipaddr6(skb, IPSET_ATTR_IP, &data->ip.in6) || nla_put_net16(skb, IPSET_ATTR_PORT, data->port) || nla_put_u8(skb, IPSET_ATTR_PROTO, data->proto)) goto nla_put_failure; return false; nla_put_failure: return true; } static void hash_ipport6_data_next(struct hash_ipport6_elem *next, const struct hash_ipport6_elem *d) { next->port = d->port; } #undef MTYPE #undef HOST_MASK #define MTYPE hash_ipport6 #define HOST_MASK 128 #define IP_SET_EMIT_CREATE #include "ip_set_hash_gen.h" static int hash_ipport6_kadt(struct ip_set *set, const struct sk_buff *skb, const struct xt_action_param *par, enum ipset_adt adt, struct ip_set_adt_opt *opt) { ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_ipport6_elem e = { .ip = { .all = { 0 } } }; struct ip_set_ext ext = IP_SET_INIT_KEXT(skb, opt, set); const struct MTYPE *h = set->data; if (!ip_set_get_ip6_port(skb, opt->flags & IPSET_DIM_TWO_SRC, &e.port, &e.proto)) return -EINVAL; ip6addrptr(skb, opt->flags & IPSET_DIM_ONE_SRC, &e.ip.in6); nf_inet_addr_mask_inplace(&e.ip, &h->bitmask); if (ipv6_addr_any(&e.ip.in6)) return -EINVAL; return adtfn(set, &e, &ext, &opt->ext, opt->cmdflags); } static int hash_ipport6_uadt(struct ip_set *set, struct nlattr *tb[], enum ipset_adt adt, u32 *lineno, u32 flags, bool retried) { const struct hash_ipport6 *h = set->data; ipset_adtfn adtfn = set->variant->adt[adt]; struct hash_ipport6_elem e = { .ip = { .all = { 0 } } }; struct ip_set_ext ext = IP_SET_INIT_UEXT(set); u32 port, port_to; bool with_ports = false; int ret; if (tb[IPSET_ATTR_LINENO]) *lineno = nla_get_u32(tb[IPSET_ATTR_LINENO]); if (unlikely(!tb[IPSET_ATTR_IP] || !ip_set_attr_netorder(tb, IPSET_ATTR_PORT) || !ip_set_optattr_netorder(tb, IPSET_ATTR_PORT_TO))) return -IPSET_ERR_PROTOCOL; if (unlikely(tb[IPSET_ATTR_IP_TO])) return -IPSET_ERR_HASH_RANGE_UNSUPPORTED; if (unlikely(tb[IPSET_ATTR_CIDR])) { u8 cidr = nla_get_u8(tb[IPSET_ATTR_CIDR]); if (cidr != HOST_MASK) return -IPSET_ERR_INVALID_CIDR; } ret = ip_set_get_ipaddr6(tb[IPSET_ATTR_IP], &e.ip); if (ret) return ret; ret = ip_set_get_extensions(set, tb, &ext); if (ret) return ret; nf_inet_addr_mask_inplace(&e.ip, &h->bitmask); if (ipv6_addr_any(&e.ip.in6)) return -EINVAL; e.port = nla_get_be16(tb[IPSET_ATTR_PORT]); if (tb[IPSET_ATTR_PROTO]) { e.proto = nla_get_u8(tb[IPSET_ATTR_PROTO]); with_ports = ip_set_proto_with_ports(e.proto); if (e.proto == 0) return -IPSET_ERR_INVALID_PROTO; } else { return -IPSET_ERR_MISSING_PROTO; } if (!(with_ports || e.proto == IPPROTO_ICMPV6)) e.port = 0; if (adt == IPSET_TEST || !with_ports || !tb[IPSET_ATTR_PORT_TO]) { ret = adtfn(set, &e, &ext, &ext, flags); return ip_set_eexist(ret, flags) ? 0 : ret; } port = ntohs(e.port); port_to = ip_set_get_h16(tb[IPSET_ATTR_PORT_TO]); if (port > port_to) swap(port, port_to); if (retried) port = ntohs(h->next.port); for (; port <= port_to; port++) { e.port = htons(port); ret = adtfn(set, &e, &ext, &ext, flags); if (ret && !ip_set_eexist(ret, flags)) return ret; ret = 0; } return ret; } static struct ip_set_type hash_ipport_type __read_mostly = { .name = "hash:ip,port", .protocol = IPSET_PROTOCOL, .features = IPSET_TYPE_IP | IPSET_TYPE_PORT, .dimension = IPSET_DIM_TWO, .family = NFPROTO_UNSPEC, .revision_min = IPSET_TYPE_REV_MIN, .revision_max = IPSET_TYPE_REV_MAX, .create_flags[IPSET_TYPE_REV_MAX] = IPSET_CREATE_FLAG_BUCKETSIZE, .create = hash_ipport_create, .create_policy = { [IPSET_ATTR_HASHSIZE] = { .type = NLA_U32 }, [IPSET_ATTR_MAXELEM] = { .type = NLA_U32 }, [IPSET_ATTR_INITVAL] = { .type = NLA_U32 }, [IPSET_ATTR_BUCKETSIZE] = { .type = NLA_U8 }, [IPSET_ATTR_RESIZE] = { .type = NLA_U8 }, [IPSET_ATTR_PROTO] = { .type = NLA_U8 }, [IPSET_ATTR_TIMEOUT] = { .type = NLA_U32 }, [IPSET_ATTR_CADT_FLAGS] = { .type = NLA_U32 }, [IPSET_ATTR_NETMASK] = { .type = NLA_U8 }, [IPSET_ATTR_BITMASK] = { .type = NLA_NESTED }, }, .adt_policy = { [IPSET_ATTR_IP] = { .type = NLA_NESTED }, [IPSET_ATTR_IP_TO] = { .type = NLA_NESTED }, [IPSET_ATTR_PORT] = { .type = NLA_U16 }, [IPSET_ATTR_PORT_TO] = { .type = NLA_U16 }, [IPSET_ATTR_CIDR] = { .type = NLA_U8 }, [IPSET_ATTR_PROTO] = { .type = NLA_U8 }, [IPSET_ATTR_TIMEOUT] = { .type = NLA_U32 }, [IPSET_ATTR_LINENO] = { .type = NLA_U32 }, [IPSET_ATTR_BYTES] = { .type = NLA_U64 }, [IPSET_ATTR_PACKETS] = { .type = NLA_U64 }, [IPSET_ATTR_COMMENT] = { .type = NLA_NUL_STRING, .len = IPSET_MAX_COMMENT_SIZE }, [IPSET_ATTR_SKBMARK] = { .type = NLA_U64 }, [IPSET_ATTR_SKBPRIO] = { .type = NLA_U32 }, [IPSET_ATTR_SKBQUEUE] = { .type = NLA_U16 }, }, .me = THIS_MODULE, }; static int __init hash_ipport_init(void) { return ip_set_type_register(&hash_ipport_type); } static void __exit hash_ipport_fini(void) { rcu_barrier(); ip_set_type_unregister(&hash_ipport_type); } module_init(hash_ipport_init); module_exit(hash_ipport_fini); |
| 48405 | 1 2 3 4 5 6 7 8 9 10 11 | /* SPDX-License-Identifier: GPL-2.0 */ #include <asm/processor.h> static inline int phys_addr_valid(resource_size_t addr) { #ifdef CONFIG_PHYS_ADDR_T_64BIT return !(addr >> boot_cpu_data.x86_phys_bits); #else return 1; #endif } |
| 376 7 378 2 376 376 376 25 25 24 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 | // SPDX-License-Identifier: GPL-2.0 /* * module.c - module sysfs fun for drivers */ #include <linux/device.h> #include <linux/module.h> #include <linux/errno.h> #include <linux/slab.h> #include <linux/string.h> #include "base.h" static char *make_driver_name(const struct device_driver *drv) { char *driver_name; driver_name = kasprintf(GFP_KERNEL, "%s:%s", drv->bus->name, drv->name); if (!driver_name) return NULL; return driver_name; } static void module_create_drivers_dir(struct module_kobject *mk) { static DEFINE_MUTEX(drivers_dir_mutex); mutex_lock(&drivers_dir_mutex); if (mk && !mk->drivers_dir) mk->drivers_dir = kobject_create_and_add("drivers", &mk->kobj); mutex_unlock(&drivers_dir_mutex); } int module_add_driver(struct module *mod, const struct device_driver *drv) { char *driver_name; struct module_kobject *mk = NULL; int ret; if (!drv) return 0; if (mod) mk = &mod->mkobj; else if (drv->mod_name) { /* Lookup or create built-in module entry in /sys/modules */ mk = lookup_or_create_module_kobject(drv->mod_name); if (mk) { /* remember our module structure */ drv->p->mkobj = mk; /* lookup_or_create_module_kobject took a reference */ kobject_put(&mk->kobj); } } if (!mk) return 0; ret = sysfs_create_link(&drv->p->kobj, &mk->kobj, "module"); if (ret) return ret; driver_name = make_driver_name(drv); if (!driver_name) { ret = -ENOMEM; goto out_remove_kobj; } module_create_drivers_dir(mk); if (!mk->drivers_dir) { ret = -EINVAL; goto out_free_driver_name; } ret = sysfs_create_link(mk->drivers_dir, &drv->p->kobj, driver_name); if (ret) goto out_remove_drivers_dir; kfree(driver_name); return 0; out_remove_drivers_dir: sysfs_remove_link(mk->drivers_dir, driver_name); out_free_driver_name: kfree(driver_name); out_remove_kobj: sysfs_remove_link(&drv->p->kobj, "module"); return ret; } void module_remove_driver(const struct device_driver *drv) { struct module_kobject *mk = NULL; char *driver_name; if (!drv) return; sysfs_remove_link(&drv->p->kobj, "module"); if (drv->owner) mk = &drv->owner->mkobj; else if (drv->p->mkobj) mk = drv->p->mkobj; if (mk && mk->drivers_dir) { driver_name = make_driver_name(drv); if (driver_name) { sysfs_remove_link(mk->drivers_dir, driver_name); kfree(driver_name); } } } |
| 6 67 164 136 2 492 9 461 597 462 12 67 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef __KVM_X86_LAPIC_H #define __KVM_X86_LAPIC_H #include <kvm/iodev.h> #include <asm/apic.h> #include <linux/kvm_host.h> #include "hyperv.h" #include "smm.h" #define KVM_APIC_INIT 0 #define KVM_APIC_SIPI 1 #define APIC_SHORT_MASK 0xc0000 #define APIC_DEST_NOSHORT 0x0 #define APIC_DEST_MASK 0x800 #define APIC_BUS_CYCLE_NS_DEFAULT 1 #define APIC_BROADCAST 0xFF #define X2APIC_BROADCAST 0xFFFFFFFFul #define X2APIC_MSR(r) (APIC_BASE_MSR + ((r) >> 4)) enum lapic_mode { LAPIC_MODE_DISABLED = 0, LAPIC_MODE_INVALID = X2APIC_ENABLE, LAPIC_MODE_XAPIC = MSR_IA32_APICBASE_ENABLE, LAPIC_MODE_X2APIC = MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE, }; enum lapic_lvt_entry { LVT_TIMER, LVT_THERMAL_MONITOR, LVT_PERFORMANCE_COUNTER, LVT_LINT0, LVT_LINT1, LVT_ERROR, LVT_CMCI, KVM_APIC_MAX_NR_LVT_ENTRIES, }; #define APIC_LVTx(x) ((x) == LVT_CMCI ? APIC_LVTCMCI : APIC_LVTT + 0x10 * (x)) struct kvm_timer { struct hrtimer timer; s64 period; /* unit: ns */ ktime_t target_expiration; u32 timer_mode; u32 timer_mode_mask; u64 tscdeadline; u64 expired_tscdeadline; u32 timer_advance_ns; atomic_t pending; /* accumulated triggered timers */ bool hv_timer_in_use; }; struct kvm_lapic { unsigned long base_address; struct kvm_io_device dev; struct kvm_timer lapic_timer; u32 divide_count; struct kvm_vcpu *vcpu; bool apicv_active; bool sw_enabled; bool irr_pending; bool lvt0_in_nmi_mode; /* Select registers in the vAPIC cannot be read/written. */ bool guest_apic_protected; /* Number of bits set in ISR. */ s16 isr_count; /* The highest vector set in ISR; if -1 - invalid, must scan ISR. */ int highest_isr_cache; /** * APIC register page. The layout matches the register layout seen by * the guest 1:1, because it is accessed by the vmx microcode. * Note: Only one register, the TPR, is used by the microcode. */ void *regs; gpa_t vapic_addr; struct gfn_to_hva_cache vapic_cache; unsigned long pending_events; unsigned int sipi_vector; int nr_lvt_entries; }; struct dest_map; int kvm_create_lapic(struct kvm_vcpu *vcpu); void kvm_free_lapic(struct kvm_vcpu *vcpu); int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu); void kvm_apic_ack_interrupt(struct kvm_vcpu *vcpu, int vector); int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); int kvm_apic_accept_events(struct kvm_vcpu *vcpu); void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu); void kvm_apic_set_version(struct kvm_vcpu *vcpu); void kvm_apic_after_set_mcg_cap(struct kvm_vcpu *vcpu); bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source, int shorthand, unsigned int dest, int dest_mode); void kvm_apic_clear_irr(struct kvm_vcpu *vcpu, int vec); bool __kvm_apic_update_irr(unsigned long *pir, void *regs, int *max_irr); bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, unsigned long *pir, int *max_irr); void kvm_apic_update_ppr(struct kvm_vcpu *vcpu); int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq, struct dest_map *dest_map); int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type); void kvm_apic_update_apicv(struct kvm_vcpu *vcpu); int kvm_alloc_apic_access_page(struct kvm *kvm); void kvm_inhibit_apic_access_page(struct kvm_vcpu *vcpu); bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src, struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map); int kvm_irq_delivery_to_apic(struct kvm *kvm, struct kvm_lapic *src, struct kvm_lapic_irq *irq, struct dest_map *dest_map); void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high); int kvm_apic_set_base(struct kvm_vcpu *vcpu, u64 value, bool host_initiated); int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s); int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s); void kvm_apic_update_hwapic_isr(struct kvm_vcpu *vcpu); int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu); void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data); void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset); void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector); int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr); void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu); void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu); int kvm_x2apic_icr_write_fast(struct kvm_lapic *apic, u64 data); int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data); int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data); int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data); int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data); int kvm_lapic_set_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len); void kvm_lapic_exit(void); u64 kvm_lapic_readable_reg_mask(struct kvm_lapic *apic); static inline void kvm_lapic_set_irr(int vec, struct kvm_lapic *apic) { apic_set_vector(vec, apic->regs + APIC_IRR); /* * irr_pending must be true if any interrupt is pending; set it after * APIC_IRR to avoid race with apic_clear_irr */ apic->irr_pending = true; } static inline u32 kvm_lapic_get_reg(struct kvm_lapic *apic, int reg_off) { return apic_get_reg(apic->regs, reg_off); } DECLARE_STATIC_KEY_FALSE(kvm_has_noapic_vcpu); static inline bool lapic_in_kernel(struct kvm_vcpu *vcpu) { if (static_branch_unlikely(&kvm_has_noapic_vcpu)) return vcpu->arch.apic; return true; } extern struct static_key_false_deferred apic_hw_disabled; static inline bool kvm_apic_hw_enabled(struct kvm_lapic *apic) { if (static_branch_unlikely(&apic_hw_disabled.key)) return apic->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE; return true; } extern struct static_key_false_deferred apic_sw_disabled; static inline bool kvm_apic_sw_enabled(struct kvm_lapic *apic) { if (static_branch_unlikely(&apic_sw_disabled.key)) return apic->sw_enabled; return true; } static inline bool kvm_apic_present(struct kvm_vcpu *vcpu) { return lapic_in_kernel(vcpu) && kvm_apic_hw_enabled(vcpu->arch.apic); } static inline int kvm_lapic_enabled(struct kvm_vcpu *vcpu) { return kvm_apic_present(vcpu) && kvm_apic_sw_enabled(vcpu->arch.apic); } static inline int apic_x2apic_mode(struct kvm_lapic *apic) { return apic->vcpu->arch.apic_base & X2APIC_ENABLE; } static inline bool kvm_vcpu_apicv_active(struct kvm_vcpu *vcpu) { return lapic_in_kernel(vcpu) && vcpu->arch.apic->apicv_active; } static inline bool kvm_apic_has_pending_init_or_sipi(struct kvm_vcpu *vcpu) { return lapic_in_kernel(vcpu) && vcpu->arch.apic->pending_events; } static inline bool kvm_apic_init_sipi_allowed(struct kvm_vcpu *vcpu) { return !is_smm(vcpu) && !kvm_x86_call(apic_init_signal_blocked)(vcpu); } static inline int kvm_lapic_latched_init(struct kvm_vcpu *vcpu) { return lapic_in_kernel(vcpu) && test_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events); } bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector); void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu); void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq, unsigned long *vcpu_bitmap); bool kvm_intr_is_single_vcpu(struct kvm *kvm, struct kvm_lapic_irq *irq, struct kvm_vcpu **dest_vcpu); void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu); void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu); void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu); bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu); void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu); static inline enum lapic_mode kvm_apic_mode(u64 apic_base) { return apic_base & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE); } static inline enum lapic_mode kvm_get_apic_mode(struct kvm_vcpu *vcpu) { return kvm_apic_mode(vcpu->arch.apic_base); } static inline u8 kvm_xapic_id(struct kvm_lapic *apic) { return kvm_lapic_get_reg(apic, APIC_ID) >> 24; } #endif |
| 3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 | /* * Copyright (c) 2004-2011 Atheros Communications Inc. * Copyright (c) 2011 Qualcomm Atheros, Inc. * * Permission to use, copy, modify, and/or distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ #ifndef HIF_OPS_H #define HIF_OPS_H #include "hif.h" #include "debug.h" static inline int hif_read_write_sync(struct ath6kl *ar, u32 addr, u8 *buf, u32 len, u32 request) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif %s sync addr 0x%x buf 0x%p len %d request 0x%x\n", (request & HIF_WRITE) ? "write" : "read", addr, buf, len, request); return ar->hif_ops->read_write_sync(ar, addr, buf, len, request); } static inline int hif_write_async(struct ath6kl *ar, u32 address, u8 *buffer, u32 length, u32 request, struct htc_packet *packet) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif write async addr 0x%x buf 0x%p len %d request 0x%x\n", address, buffer, length, request); return ar->hif_ops->write_async(ar, address, buffer, length, request, packet); } static inline void ath6kl_hif_irq_enable(struct ath6kl *ar) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif irq enable\n"); return ar->hif_ops->irq_enable(ar); } static inline void ath6kl_hif_irq_disable(struct ath6kl *ar) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif irq disable\n"); return ar->hif_ops->irq_disable(ar); } static inline struct hif_scatter_req *hif_scatter_req_get(struct ath6kl *ar) { return ar->hif_ops->scatter_req_get(ar); } static inline void hif_scatter_req_add(struct ath6kl *ar, struct hif_scatter_req *s_req) { return ar->hif_ops->scatter_req_add(ar, s_req); } static inline int ath6kl_hif_enable_scatter(struct ath6kl *ar) { return ar->hif_ops->enable_scatter(ar); } static inline int ath6kl_hif_scat_req_rw(struct ath6kl *ar, struct hif_scatter_req *scat_req) { return ar->hif_ops->scat_req_rw(ar, scat_req); } static inline void ath6kl_hif_cleanup_scatter(struct ath6kl *ar) { return ar->hif_ops->cleanup_scatter(ar); } static inline int ath6kl_hif_suspend(struct ath6kl *ar, struct cfg80211_wowlan *wow) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif suspend\n"); return ar->hif_ops->suspend(ar, wow); } /* * Read from the ATH6KL through its diagnostic window. No cooperation from * the Target is required for this. */ static inline int ath6kl_hif_diag_read32(struct ath6kl *ar, u32 address, u32 *value) { return ar->hif_ops->diag_read32(ar, address, value); } /* * Write to the ATH6KL through its diagnostic window. No cooperation from * the Target is required for this. */ static inline int ath6kl_hif_diag_write32(struct ath6kl *ar, u32 address, __le32 value) { return ar->hif_ops->diag_write32(ar, address, value); } static inline int ath6kl_hif_bmi_read(struct ath6kl *ar, u8 *buf, u32 len) { return ar->hif_ops->bmi_read(ar, buf, len); } static inline int ath6kl_hif_bmi_write(struct ath6kl *ar, u8 *buf, u32 len) { return ar->hif_ops->bmi_write(ar, buf, len); } static inline int ath6kl_hif_resume(struct ath6kl *ar) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif resume\n"); return ar->hif_ops->resume(ar); } static inline int ath6kl_hif_power_on(struct ath6kl *ar) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif power on\n"); return ar->hif_ops->power_on(ar); } static inline int ath6kl_hif_power_off(struct ath6kl *ar) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif power off\n"); return ar->hif_ops->power_off(ar); } static inline void ath6kl_hif_stop(struct ath6kl *ar) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif stop\n"); ar->hif_ops->stop(ar); } static inline int ath6kl_hif_pipe_send(struct ath6kl *ar, u8 pipe, struct sk_buff *hdr_buf, struct sk_buff *buf) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif pipe send\n"); return ar->hif_ops->pipe_send(ar, pipe, hdr_buf, buf); } static inline void ath6kl_hif_pipe_get_default(struct ath6kl *ar, u8 *ul_pipe, u8 *dl_pipe) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif pipe get default\n"); ar->hif_ops->pipe_get_default(ar, ul_pipe, dl_pipe); } static inline int ath6kl_hif_pipe_map_service(struct ath6kl *ar, u16 service_id, u8 *ul_pipe, u8 *dl_pipe) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif pipe get default\n"); return ar->hif_ops->pipe_map_service(ar, service_id, ul_pipe, dl_pipe); } static inline u16 ath6kl_hif_pipe_get_free_queue_number(struct ath6kl *ar, u8 pipe) { ath6kl_dbg(ATH6KL_DBG_HIF, "hif pipe get free queue number\n"); return ar->hif_ops->pipe_get_free_queue_number(ar, pipe); } #endif |
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 | /* SPDX-License-Identifier: GPL-2.0 */ /* * befs.h * * Copyright (C) 2001-2002 Will Dyson <will_dyson@pobox.com> * Copyright (C) 1999 Makoto Kato (m_kato@ga2.so-net.ne.jp) */ #ifndef _LINUX_BEFS_H #define _LINUX_BEFS_H #include "befs_fs_types.h" /* used in debug.c */ #define BEFS_VERSION "0.9.3" typedef u64 befs_blocknr_t; /* * BeFS in memory structures */ struct befs_mount_options { kgid_t gid; kuid_t uid; int use_gid; int use_uid; int debug; char *iocharset; }; struct befs_sb_info { u32 magic1; u32 block_size; u32 block_shift; int byte_order; befs_off_t num_blocks; befs_off_t used_blocks; u32 inode_size; u32 magic2; /* Allocation group information */ u32 blocks_per_ag; u32 ag_shift; u32 num_ags; /* State of the superblock */ u32 flags; /* Journal log entry */ befs_block_run log_blocks; befs_off_t log_start; befs_off_t log_end; befs_inode_addr root_dir; befs_inode_addr indices; u32 magic3; struct befs_mount_options mount_opts; struct nls_table *nls; }; struct befs_inode_info { u32 i_flags; u32 i_type; befs_inode_addr i_inode_num; befs_inode_addr i_parent; befs_inode_addr i_attribute; union { befs_data_stream ds; char symlink[BEFS_SYMLINK_LEN]; } i_data; struct inode vfs_inode; }; enum befs_err { BEFS_OK, BEFS_ERR, BEFS_BAD_INODE, BEFS_BT_END, BEFS_BT_EMPTY, BEFS_BT_MATCH, BEFS_BT_OVERFLOW, BEFS_BT_NOT_FOUND }; /****************************/ /* debug.c */ __printf(2, 3) void befs_error(const struct super_block *sb, const char *fmt, ...); __printf(2, 3) void befs_warning(const struct super_block *sb, const char *fmt, ...); __printf(2, 3) void befs_debug(const struct super_block *sb, const char *fmt, ...); void befs_dump_super_block(const struct super_block *sb, befs_super_block *); void befs_dump_inode(const struct super_block *sb, befs_inode *); void befs_dump_index_entry(const struct super_block *sb, befs_disk_btree_super *); void befs_dump_index_node(const struct super_block *sb, befs_btree_nodehead *); /****************************/ /* Gets a pointer to the private portion of the super_block * structure from the public part */ static inline struct befs_sb_info * BEFS_SB(const struct super_block *super) { return (struct befs_sb_info *) super->s_fs_info; } static inline struct befs_inode_info * BEFS_I(const struct inode *inode) { return container_of(inode, struct befs_inode_info, vfs_inode); } static inline befs_blocknr_t iaddr2blockno(struct super_block *sb, const befs_inode_addr *iaddr) { return ((iaddr->allocation_group << BEFS_SB(sb)->ag_shift) + iaddr->start); } static inline befs_inode_addr blockno2iaddr(struct super_block *sb, befs_blocknr_t blockno) { befs_inode_addr iaddr; iaddr.allocation_group = blockno >> BEFS_SB(sb)->ag_shift; iaddr.start = blockno - (iaddr.allocation_group << BEFS_SB(sb)->ag_shift); iaddr.len = 1; return iaddr; } static inline unsigned int befs_iaddrs_per_block(struct super_block *sb) { return BEFS_SB(sb)->block_size / sizeof(befs_disk_inode_addr); } #include "endian.h" #endif /* _LINUX_BEFS_H */ |
| 64 5 14 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 | /* SPDX-License-Identifier: GPL-2.0 */ /* * This is <linux/capability.h> * * Andrew G. Morgan <morgan@kernel.org> * Alexander Kjeldaas <astor@guardian.no> * with help from Aleph1, Roland Buresund and Andrew Main. * * See here for the libcap library ("POSIX draft" compliance): * * ftp://www.kernel.org/pub/linux/libs/security/linux-privs/kernel-2.6/ */ #ifndef _LINUX_CAPABILITY_H #define _LINUX_CAPABILITY_H #include <uapi/linux/capability.h> #include <linux/uidgid.h> #include <linux/bits.h> #define _KERNEL_CAPABILITY_VERSION _LINUX_CAPABILITY_VERSION_3 extern int file_caps_enabled; typedef struct { u64 val; } kernel_cap_t; /* same as vfs_ns_cap_data but in cpu endian and always filled completely */ struct cpu_vfs_cap_data { __u32 magic_etc; kuid_t rootid; kernel_cap_t permitted; kernel_cap_t inheritable; }; #define _USER_CAP_HEADER_SIZE (sizeof(struct __user_cap_header_struct)) #define _KERNEL_CAP_T_SIZE (sizeof(kernel_cap_t)) struct file; struct inode; struct dentry; struct task_struct; struct user_namespace; struct mnt_idmap; /* * CAP_FS_MASK and CAP_NFSD_MASKS: * * The fs mask is all the privileges that fsuid==0 historically meant. * At one time in the past, that included CAP_MKNOD and CAP_LINUX_IMMUTABLE. * * It has never meant setting security.* and trusted.* xattrs. * * We could also define fsmask as follows: * 1. CAP_FS_MASK is the privilege to bypass all fs-related DAC permissions * 2. The security.* and trusted.* xattrs are fs-related MAC permissions */ # define CAP_FS_MASK (BIT_ULL(CAP_CHOWN) \ | BIT_ULL(CAP_MKNOD) \ | BIT_ULL(CAP_DAC_OVERRIDE) \ | BIT_ULL(CAP_DAC_READ_SEARCH) \ | BIT_ULL(CAP_FOWNER) \ | BIT_ULL(CAP_FSETID) \ | BIT_ULL(CAP_MAC_OVERRIDE)) #define CAP_VALID_MASK (BIT_ULL(CAP_LAST_CAP+1)-1) # define CAP_EMPTY_SET ((kernel_cap_t) { 0 }) # define CAP_FULL_SET ((kernel_cap_t) { CAP_VALID_MASK }) # define CAP_FS_SET ((kernel_cap_t) { CAP_FS_MASK | BIT_ULL(CAP_LINUX_IMMUTABLE) }) # define CAP_NFSD_SET ((kernel_cap_t) { CAP_FS_MASK | BIT_ULL(CAP_SYS_RESOURCE) }) # define cap_clear(c) do { (c).val = 0; } while (0) #define cap_raise(c, flag) ((c).val |= BIT_ULL(flag)) #define cap_lower(c, flag) ((c).val &= ~BIT_ULL(flag)) #define cap_raised(c, flag) (((c).val & BIT_ULL(flag)) != 0) static inline kernel_cap_t cap_combine(const kernel_cap_t a, const kernel_cap_t b) { return (kernel_cap_t) { a.val | b.val }; } static inline kernel_cap_t cap_intersect(const kernel_cap_t a, const kernel_cap_t b) { return (kernel_cap_t) { a.val & b.val }; } static inline kernel_cap_t cap_drop(const kernel_cap_t a, const kernel_cap_t drop) { return (kernel_cap_t) { a.val &~ drop.val }; } static inline bool cap_isclear(const kernel_cap_t a) { return !a.val; } static inline bool cap_isidentical(const kernel_cap_t a, const kernel_cap_t b) { return a.val == b.val; } /* * Check if "a" is a subset of "set". * return true if ALL of the capabilities in "a" are also in "set" * cap_issubset(0101, 1111) will return true * return false if ANY of the capabilities in "a" are not in "set" * cap_issubset(1111, 0101) will return false */ static inline bool cap_issubset(const kernel_cap_t a, const kernel_cap_t set) { return !(a.val & ~set.val); } /* Used to decide between falling back on the old suser() or fsuser(). */ static inline kernel_cap_t cap_drop_fs_set(const kernel_cap_t a) { return cap_drop(a, CAP_FS_SET); } static inline kernel_cap_t cap_raise_fs_set(const kernel_cap_t a, const kernel_cap_t permitted) { return cap_combine(a, cap_intersect(permitted, CAP_FS_SET)); } static inline kernel_cap_t cap_drop_nfsd_set(const kernel_cap_t a) { return cap_drop(a, CAP_NFSD_SET); } static inline kernel_cap_t cap_raise_nfsd_set(const kernel_cap_t a, const kernel_cap_t permitted) { return cap_combine(a, cap_intersect(permitted, CAP_NFSD_SET)); } #ifdef CONFIG_MULTIUSER extern bool has_ns_capability(struct task_struct *t, struct user_namespace *ns, int cap); extern bool has_capability_noaudit(struct task_struct *t, int cap); extern bool has_ns_capability_noaudit(struct task_struct *t, struct user_namespace *ns, int cap); extern bool capable(int cap); extern bool ns_capable(struct user_namespace *ns, int cap); extern bool ns_capable_noaudit(struct user_namespace *ns, int cap); extern bool ns_capable_setid(struct user_namespace *ns, int cap); #else static inline bool has_ns_capability(struct task_struct *t, struct user_namespace *ns, int cap) { return true; } static inline bool has_capability_noaudit(struct task_struct *t, int cap) { return true; } static inline bool has_ns_capability_noaudit(struct task_struct *t, struct user_namespace *ns, int cap) { return true; } static inline bool capable(int cap) { return true; } static inline bool ns_capable(struct user_namespace *ns, int cap) { return true; } static inline bool ns_capable_noaudit(struct user_namespace *ns, int cap) { return true; } static inline bool ns_capable_setid(struct user_namespace *ns, int cap) { return true; } #endif /* CONFIG_MULTIUSER */ bool privileged_wrt_inode_uidgid(struct user_namespace *ns, struct mnt_idmap *idmap, const struct inode *inode); bool capable_wrt_inode_uidgid(struct mnt_idmap *idmap, const struct inode *inode, int cap); extern bool file_ns_capable(const struct file *file, struct user_namespace *ns, int cap); extern bool ptracer_capable(struct task_struct *tsk, struct user_namespace *ns); static inline bool perfmon_capable(void) { return capable(CAP_PERFMON) || capable(CAP_SYS_ADMIN); } static inline bool bpf_capable(void) { return capable(CAP_BPF) || capable(CAP_SYS_ADMIN); } static inline bool checkpoint_restore_ns_capable(struct user_namespace *ns) { return ns_capable(ns, CAP_CHECKPOINT_RESTORE) || ns_capable(ns, CAP_SYS_ADMIN); } /* audit system wants to get cap info from files as well */ int get_vfs_caps_from_disk(struct mnt_idmap *idmap, const struct dentry *dentry, struct cpu_vfs_cap_data *cpu_caps); int cap_convert_nscap(struct mnt_idmap *idmap, struct dentry *dentry, const void **ivalue, size_t size); #endif /* !_LINUX_CAPABILITY_H */ |
| 37 590 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 | /* * Linux Security Module interfaces * * Copyright (C) 2001 WireX Communications, Inc <chris@wirex.com> * Copyright (C) 2001 Greg Kroah-Hartman <greg@kroah.com> * Copyright (C) 2001 Networks Associates Technology, Inc <ssmalley@nai.com> * Copyright (C) 2001 James Morris <jmorris@intercode.com.au> * Copyright (C) 2001 Silicon Graphics, Inc. (Trust Technology Group) * Copyright (C) 2015 Intel Corporation. * Copyright (C) 2015 Casey Schaufler <casey@schaufler-ca.com> * Copyright (C) 2016 Mellanox Techonologies * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * Due to this file being licensed under the GPL there is controversy over * whether this permits you to write a module that #includes this file * without placing your module under the GPL. Please consult a lawyer for * advice before doing this. * */ #ifndef __LINUX_LSM_HOOKS_H #define __LINUX_LSM_HOOKS_H #include <uapi/linux/lsm.h> #include <linux/security.h> #include <linux/init.h> #include <linux/rculist.h> #include <linux/xattr.h> #include <linux/static_call.h> #include <linux/unroll.h> #include <linux/jump_label.h> #include <linux/lsm_count.h> union security_list_options { #define LSM_HOOK(RET, DEFAULT, NAME, ...) RET (*NAME)(__VA_ARGS__); #include "lsm_hook_defs.h" #undef LSM_HOOK void *lsm_func_addr; }; /* * @key: static call key as defined by STATIC_CALL_KEY * @trampoline: static call trampoline as defined by STATIC_CALL_TRAMP * @hl: The security_hook_list as initialized by the owning LSM. * @active: Enabled when the static call has an LSM hook associated. */ struct lsm_static_call { struct static_call_key *key; void *trampoline; struct security_hook_list *hl; /* this needs to be true or false based on what the key defaults to */ struct static_key_false *active; } __randomize_layout; /* * Table of the static calls for each LSM hook. * Once the LSMs are initialized, their callbacks will be copied to these * tables such that the calls are filled backwards (from last to first). * This way, we can jump directly to the first used static call, and execute * all of them after. This essentially makes the entry point * dynamic to adapt the number of static calls to the number of callbacks. */ struct lsm_static_calls_table { #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ struct lsm_static_call NAME[MAX_LSM_COUNT]; #include <linux/lsm_hook_defs.h> #undef LSM_HOOK } __packed __randomize_layout; /** * struct lsm_id - Identify a Linux Security Module. * @lsm: name of the LSM, must be approved by the LSM maintainers * @id: LSM ID number from uapi/linux/lsm.h * * Contains the information that identifies the LSM. */ struct lsm_id { const char *name; u64 id; }; /* * Security module hook list structure. * For use with generic list macros for common operations. * * struct security_hook_list - Contents of a cacheable, mappable object. * @scalls: The beginning of the array of static calls assigned to this hook. * @hook: The callback for the hook. * @lsm: The name of the lsm that owns this hook. */ struct security_hook_list { struct lsm_static_call *scalls; union security_list_options hook; const struct lsm_id *lsmid; } __randomize_layout; /* * Security blob size or offset data. */ struct lsm_blob_sizes { int lbs_cred; int lbs_file; int lbs_ib; int lbs_inode; int lbs_sock; int lbs_superblock; int lbs_ipc; int lbs_key; int lbs_msg_msg; int lbs_perf_event; int lbs_task; int lbs_xattr_count; /* number of xattr slots in new_xattrs array */ int lbs_tun_dev; int lbs_bdev; int lbs_bpf_map; int lbs_bpf_prog; int lbs_bpf_token; }; /* * LSM_RET_VOID is used as the default value in LSM_HOOK definitions for void * LSM hooks (in include/linux/lsm_hook_defs.h). */ #define LSM_RET_VOID ((void) 0) /* * Initializing a security_hook_list structure takes * up a lot of space in a source file. This macro takes * care of the common case and reduces the amount of * text involved. */ #define LSM_HOOK_INIT(NAME, HOOK) \ { \ .scalls = static_calls_table.NAME, \ .hook = { .NAME = HOOK } \ } extern void security_add_hooks(struct security_hook_list *hooks, int count, const struct lsm_id *lsmid); #define LSM_FLAG_LEGACY_MAJOR BIT(0) #define LSM_FLAG_EXCLUSIVE BIT(1) enum lsm_order { LSM_ORDER_FIRST = -1, /* This is only for capabilities. */ LSM_ORDER_MUTABLE = 0, LSM_ORDER_LAST = 1, /* This is only for integrity. */ }; struct lsm_info { const char *name; /* Required. */ enum lsm_order order; /* Optional: default is LSM_ORDER_MUTABLE */ unsigned long flags; /* Optional: flags describing LSM */ int *enabled; /* Optional: controlled by CONFIG_LSM */ int (*init)(void); /* Required. */ struct lsm_blob_sizes *blobs; /* Optional: for blob sharing. */ }; #define DEFINE_LSM(lsm) \ static struct lsm_info __lsm_##lsm \ __used __section(".lsm_info.init") \ __aligned(sizeof(unsigned long)) #define DEFINE_EARLY_LSM(lsm) \ static struct lsm_info __early_lsm_##lsm \ __used __section(".early_lsm_info.init") \ __aligned(sizeof(unsigned long)) /* DO NOT tamper with these variables outside of the LSM framework */ extern char *lsm_names; extern struct lsm_static_calls_table static_calls_table __ro_after_init; extern struct lsm_info __start_lsm_info[], __end_lsm_info[]; extern struct lsm_info __start_early_lsm_info[], __end_early_lsm_info[]; /** * lsm_get_xattr_slot - Return the next available slot and increment the index * @xattrs: array storing LSM-provided xattrs * @xattr_count: number of already stored xattrs (updated) * * Retrieve the first available slot in the @xattrs array to fill with an xattr, * and increment @xattr_count. * * Return: The slot to fill in @xattrs if non-NULL, NULL otherwise. */ static inline struct xattr *lsm_get_xattr_slot(struct xattr *xattrs, int *xattr_count) { if (unlikely(!xattrs)) return NULL; return &xattrs[(*xattr_count)++]; } #endif /* ! __LINUX_LSM_HOOKS_H */ |
| 8 8 9 9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 | // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. */ #include "ratelimiter.h" #include <linux/siphash.h> #include <linux/mm.h> #include <linux/slab.h> #include <net/ip.h> static struct kmem_cache *entry_cache; static hsiphash_key_t key; static spinlock_t table_lock = __SPIN_LOCK_UNLOCKED("ratelimiter_table_lock"); static DEFINE_MUTEX(init_lock); static u64 init_refcnt; /* Protected by init_lock, hence not atomic. */ static atomic_t total_entries = ATOMIC_INIT(0); static unsigned int max_entries, table_size; static void wg_ratelimiter_gc_entries(struct work_struct *); static DECLARE_DEFERRABLE_WORK(gc_work, wg_ratelimiter_gc_entries); static struct hlist_head *table_v4; #if IS_ENABLED(CONFIG_IPV6) static struct hlist_head *table_v6; #endif struct ratelimiter_entry { u64 last_time_ns, tokens, ip; void *net; spinlock_t lock; struct hlist_node hash; struct rcu_head rcu; }; enum { PACKETS_PER_SECOND = 20, PACKETS_BURSTABLE = 5, PACKET_COST = NSEC_PER_SEC / PACKETS_PER_SECOND, TOKEN_MAX = PACKET_COST * PACKETS_BURSTABLE }; static void entry_free(struct rcu_head *rcu) { kmem_cache_free(entry_cache, container_of(rcu, struct ratelimiter_entry, rcu)); atomic_dec(&total_entries); } static void entry_uninit(struct ratelimiter_entry *entry) { hlist_del_rcu(&entry->hash); call_rcu(&entry->rcu, entry_free); } /* Calling this function with a NULL work uninits all entries. */ static void wg_ratelimiter_gc_entries(struct work_struct *work) { const u64 now = ktime_get_coarse_boottime_ns(); struct ratelimiter_entry *entry; struct hlist_node *temp; unsigned int i; for (i = 0; i < table_size; ++i) { spin_lock(&table_lock); hlist_for_each_entry_safe(entry, temp, &table_v4[i], hash) { if (unlikely(!work) || now - entry->last_time_ns > NSEC_PER_SEC) entry_uninit(entry); } #if IS_ENABLED(CONFIG_IPV6) hlist_for_each_entry_safe(entry, temp, &table_v6[i], hash) { if (unlikely(!work) || now - entry->last_time_ns > NSEC_PER_SEC) entry_uninit(entry); } #endif spin_unlock(&table_lock); if (likely(work)) cond_resched(); } if (likely(work)) queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); } bool wg_ratelimiter_allow(struct sk_buff *skb, struct net *net) { /* We only take the bottom half of the net pointer, so that we can hash * 3 words in the end. This way, siphash's len param fits into the final * u32, and we don't incur an extra round. */ const u32 net_word = (unsigned long)net; struct ratelimiter_entry *entry; struct hlist_head *bucket; u64 ip; if (skb->protocol == htons(ETH_P_IP)) { ip = (u64 __force)ip_hdr(skb)->saddr; bucket = &table_v4[hsiphash_2u32(net_word, ip, &key) & (table_size - 1)]; } #if IS_ENABLED(CONFIG_IPV6) else if (skb->protocol == htons(ETH_P_IPV6)) { /* Only use 64 bits, so as to ratelimit the whole /64. */ memcpy(&ip, &ipv6_hdr(skb)->saddr, sizeof(ip)); bucket = &table_v6[hsiphash_3u32(net_word, ip >> 32, ip, &key) & (table_size - 1)]; } #endif else return false; rcu_read_lock(); hlist_for_each_entry_rcu(entry, bucket, hash) { if (entry->net == net && entry->ip == ip) { u64 now, tokens; bool ret; /* Quasi-inspired by nft_limit.c, but this is actually a * slightly different algorithm. Namely, we incorporate * the burst as part of the maximum tokens, rather than * as part of the rate. */ spin_lock(&entry->lock); now = ktime_get_coarse_boottime_ns(); tokens = min_t(u64, TOKEN_MAX, entry->tokens + now - entry->last_time_ns); entry->last_time_ns = now; ret = tokens >= PACKET_COST; entry->tokens = ret ? tokens - PACKET_COST : tokens; spin_unlock(&entry->lock); rcu_read_unlock(); return ret; } } rcu_read_unlock(); if (atomic_inc_return(&total_entries) > max_entries) goto err_oom; entry = kmem_cache_alloc(entry_cache, GFP_KERNEL); if (unlikely(!entry)) goto err_oom; entry->net = net; entry->ip = ip; INIT_HLIST_NODE(&entry->hash); spin_lock_init(&entry->lock); entry->last_time_ns = ktime_get_coarse_boottime_ns(); entry->tokens = TOKEN_MAX - PACKET_COST; spin_lock(&table_lock); hlist_add_head_rcu(&entry->hash, bucket); spin_unlock(&table_lock); return true; err_oom: atomic_dec(&total_entries); return false; } int wg_ratelimiter_init(void) { mutex_lock(&init_lock); if (++init_refcnt != 1) goto out; entry_cache = KMEM_CACHE(ratelimiter_entry, 0); if (!entry_cache) goto err; /* xt_hashlimit.c uses a slightly different algorithm for ratelimiting, * but what it shares in common is that it uses a massive hashtable. So, * we borrow their wisdom about good table sizes on different systems * dependent on RAM. This calculation here comes from there. */ table_size = (totalram_pages() > (1U << 30) / PAGE_SIZE) ? 8192 : max_t(unsigned long, 16, roundup_pow_of_two( (totalram_pages() << PAGE_SHIFT) / (1U << 14) / sizeof(struct hlist_head))); max_entries = table_size * 8; table_v4 = kvcalloc(table_size, sizeof(*table_v4), GFP_KERNEL); if (unlikely(!table_v4)) goto err_kmemcache; #if IS_ENABLED(CONFIG_IPV6) table_v6 = kvcalloc(table_size, sizeof(*table_v6), GFP_KERNEL); if (unlikely(!table_v6)) { kvfree(table_v4); goto err_kmemcache; } #endif queue_delayed_work(system_power_efficient_wq, &gc_work, HZ); get_random_bytes(&key, sizeof(key)); out: mutex_unlock(&init_lock); return 0; err_kmemcache: kmem_cache_destroy(entry_cache); err: --init_refcnt; mutex_unlock(&init_lock); return -ENOMEM; } void wg_ratelimiter_uninit(void) { mutex_lock(&init_lock); if (!init_refcnt || --init_refcnt) goto out; cancel_delayed_work_sync(&gc_work); wg_ratelimiter_gc_entries(NULL); rcu_barrier(); kvfree(table_v4); #if IS_ENABLED(CONFIG_IPV6) kvfree(table_v6); #endif kmem_cache_destroy(entry_cache); out: mutex_unlock(&init_lock); } #include "selftest/ratelimiter.c" |
| 9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 | /* SPDX-License-Identifier: GPL-2.0-only */ /* * File: pep.h * * Phonet Pipe End Point sockets definitions * * Copyright (C) 2008 Nokia Corporation. */ #ifndef NET_PHONET_PEP_H #define NET_PHONET_PEP_H #include <linux/skbuff.h> #include <net/phonet/phonet.h> struct pep_sock { struct pn_sock pn_sk; /* XXX: union-ify listening vs connected stuff ? */ /* Listening socket stuff: */ struct hlist_head hlist; /* Connected socket stuff: */ struct sock *listener; struct sk_buff_head ctrlreq_queue; #define PNPIPE_CTRLREQ_MAX 10 atomic_t tx_credits; int ifindex; u16 peer_type; /* peer type/subtype */ u8 pipe_handle; u8 rx_credits; u8 rx_fc; /* RX flow control */ u8 tx_fc; /* TX flow control */ u8 init_enable; /* auto-enable at creation */ u8 aligned; }; static inline struct pep_sock *pep_sk(struct sock *sk) { return (struct pep_sock *)sk; } extern const struct proto_ops phonet_stream_ops; /* Pipe protocol definitions */ struct pnpipehdr { u8 utid; /* transaction ID */ u8 message_id; u8 pipe_handle; union { u8 state_after_connect; /* connect request */ u8 state_after_reset; /* reset request */ u8 error_code; /* any response */ u8 pep_type; /* status indication */ u8 data0; /* anything else */ }; u8 data[]; }; #define other_pep_type data[0] static inline struct pnpipehdr *pnp_hdr(struct sk_buff *skb) { return (struct pnpipehdr *)skb_transport_header(skb); } #define MAX_PNPIPE_HEADER (MAX_PHONET_HEADER + 4) enum { PNS_PIPE_CREATE_REQ = 0x00, PNS_PIPE_CREATE_RESP, PNS_PIPE_REMOVE_REQ, PNS_PIPE_REMOVE_RESP, PNS_PIPE_DATA = 0x20, PNS_PIPE_ALIGNED_DATA, PNS_PEP_CONNECT_REQ = 0x40, PNS_PEP_CONNECT_RESP, PNS_PEP_DISCONNECT_REQ, PNS_PEP_DISCONNECT_RESP, PNS_PEP_RESET_REQ, PNS_PEP_RESET_RESP, PNS_PEP_ENABLE_REQ, PNS_PEP_ENABLE_RESP, PNS_PEP_CTRL_REQ, PNS_PEP_CTRL_RESP, PNS_PEP_DISABLE_REQ = 0x4C, PNS_PEP_DISABLE_RESP, PNS_PEP_STATUS_IND = 0x60, PNS_PIPE_CREATED_IND, PNS_PIPE_RESET_IND = 0x63, PNS_PIPE_ENABLED_IND, PNS_PIPE_REDIRECTED_IND, PNS_PIPE_DISABLED_IND = 0x66, }; #define PN_PIPE_INVALID_HANDLE 0xff #define PN_PEP_TYPE_COMMON 0x00 /* Phonet pipe status indication */ enum { PN_PEP_IND_FLOW_CONTROL, PN_PEP_IND_ID_MCFC_GRANT_CREDITS, }; /* Phonet pipe error codes */ enum { PN_PIPE_NO_ERROR, PN_PIPE_ERR_INVALID_PARAM, PN_PIPE_ERR_INVALID_HANDLE, PN_PIPE_ERR_INVALID_CTRL_ID, PN_PIPE_ERR_NOT_ALLOWED, PN_PIPE_ERR_PEP_IN_USE, PN_PIPE_ERR_OVERLOAD, PN_PIPE_ERR_DEV_DISCONNECTED, PN_PIPE_ERR_TIMEOUT, PN_PIPE_ERR_ALL_PIPES_IN_USE, PN_PIPE_ERR_GENERAL, PN_PIPE_ERR_NOT_SUPPORTED, }; /* Phonet pipe states */ enum { PN_PIPE_DISABLE, PN_PIPE_ENABLE, }; /* Phonet pipe sub-block types */ enum { PN_PIPE_SB_CREATE_REQ_PEP_SUB_TYPE, PN_PIPE_SB_CONNECT_REQ_PEP_SUB_TYPE, PN_PIPE_SB_REDIRECT_REQ_PEP_SUB_TYPE, PN_PIPE_SB_NEGOTIATED_FC, PN_PIPE_SB_REQUIRED_FC_TX, PN_PIPE_SB_PREFERRED_FC_RX, PN_PIPE_SB_ALIGNED_DATA, }; /* Phonet pipe flow control models */ enum { PN_NO_FLOW_CONTROL, PN_LEGACY_FLOW_CONTROL, PN_ONE_CREDIT_FLOW_CONTROL, PN_MULTI_CREDIT_FLOW_CONTROL, PN_MAX_FLOW_CONTROL, }; #define pn_flow_safe(fc) ((fc) >> 1) /* Phonet pipe flow control states */ enum { PEP_IND_EMPTY, PEP_IND_BUSY, PEP_IND_READY, }; #endif |
| 14 14 14 14 14 14 14 14 5 2 3 3 2 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 | // SPDX-License-Identifier: GPL-2.0-only #include <linux/bcd.h> #include <linux/delay.h> #include <linux/export.h> #include <linux/mc146818rtc.h> #ifdef CONFIG_ACPI #include <linux/acpi.h> #endif #define UIP_RECHECK_DELAY 100 /* usec */ #define UIP_RECHECK_DELAY_MS (USEC_PER_MSEC / UIP_RECHECK_DELAY) #define UIP_RECHECK_LOOPS_MS(x) (x / UIP_RECHECK_DELAY_MS) /* * Execute a function while the UIP (Update-in-progress) bit of the RTC is * unset. The timeout is configurable by the caller in ms. * * Warning: callback may be executed more then once. */ bool mc146818_avoid_UIP(void (*callback)(unsigned char seconds, void *param), int timeout, void *param) { int i; unsigned long flags; unsigned char seconds; for (i = 0; UIP_RECHECK_LOOPS_MS(i) < timeout; i++) { spin_lock_irqsave(&rtc_lock, flags); /* * Check whether there is an update in progress during which the * readout is unspecified. The maximum update time is ~2ms. Poll * for completion. * * Store the second value before checking UIP so a long lasting * NMI which happens to hit after the UIP check cannot make * an update cycle invisible. */ seconds = CMOS_READ(RTC_SECONDS); if (CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) { spin_unlock_irqrestore(&rtc_lock, flags); udelay(UIP_RECHECK_DELAY); continue; } /* Revalidate the above readout */ if (seconds != CMOS_READ(RTC_SECONDS)) { spin_unlock_irqrestore(&rtc_lock, flags); continue; } if (callback) callback(seconds, param); /* * Check for the UIP bit again. If it is set now then * the above values may contain garbage. */ if (CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) { spin_unlock_irqrestore(&rtc_lock, flags); udelay(UIP_RECHECK_DELAY); continue; } /* * A NMI might have interrupted the above sequence so check * whether the seconds value has changed which indicates that * the NMI took longer than the UIP bit was set. Unlikely, but * possible and there is also virt... */ if (seconds != CMOS_READ(RTC_SECONDS)) { spin_unlock_irqrestore(&rtc_lock, flags); continue; } spin_unlock_irqrestore(&rtc_lock, flags); if (UIP_RECHECK_LOOPS_MS(i) >= 100) pr_warn("Reading current time from RTC took around %li ms\n", UIP_RECHECK_LOOPS_MS(i)); return true; } return false; } EXPORT_SYMBOL_GPL(mc146818_avoid_UIP); /* * If the UIP (Update-in-progress) bit of the RTC is set for more then * 10ms, the RTC is apparently broken or not present. */ bool mc146818_does_rtc_work(void) { return mc146818_avoid_UIP(NULL, 1000, NULL); } EXPORT_SYMBOL_GPL(mc146818_does_rtc_work); struct mc146818_get_time_callback_param { struct rtc_time *time; unsigned char ctrl; #ifdef CONFIG_ACPI unsigned char century; #endif #ifdef CONFIG_MACH_DECSTATION unsigned int real_year; #endif }; static void mc146818_get_time_callback(unsigned char seconds, void *param_in) { struct mc146818_get_time_callback_param *p = param_in; /* * Only the values that we read from the RTC are set. We leave * tm_wday, tm_yday and tm_isdst untouched. Even though the * RTC has RTC_DAY_OF_WEEK, we ignore it, as it is only updated * by the RTC when initially set to a non-zero value. */ p->time->tm_sec = seconds; p->time->tm_min = CMOS_READ(RTC_MINUTES); p->time->tm_hour = CMOS_READ(RTC_HOURS); p->time->tm_mday = CMOS_READ(RTC_DAY_OF_MONTH); p->time->tm_mon = CMOS_READ(RTC_MONTH); p->time->tm_year = CMOS_READ(RTC_YEAR); #ifdef CONFIG_MACH_DECSTATION p->real_year = CMOS_READ(RTC_DEC_YEAR); #endif #ifdef CONFIG_ACPI if (acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID && acpi_gbl_FADT.century) { p->century = CMOS_READ(acpi_gbl_FADT.century); } else { p->century = 0; } #endif p->ctrl = CMOS_READ(RTC_CONTROL); } /** * mc146818_get_time - Get the current time from the RTC * @time: pointer to struct rtc_time to store the current time * @timeout: timeout value in ms * * This function reads the current time from the RTC and stores it in the * provided struct rtc_time. The timeout parameter specifies the maximum * time to wait for the RTC to become ready. * * Return: 0 on success, -ETIMEDOUT if the RTC did not become ready within * the specified timeout, or another error code if an error occurred. */ int mc146818_get_time(struct rtc_time *time, int timeout) { struct mc146818_get_time_callback_param p = { .time = time }; if (!mc146818_avoid_UIP(mc146818_get_time_callback, timeout, &p)) { memset(time, 0, sizeof(*time)); return -ETIMEDOUT; } if (!(p.ctrl & RTC_DM_BINARY) || RTC_ALWAYS_BCD) { time->tm_sec = bcd2bin(time->tm_sec); time->tm_min = bcd2bin(time->tm_min); time->tm_hour = bcd2bin(time->tm_hour); time->tm_mday = bcd2bin(time->tm_mday); time->tm_mon = bcd2bin(time->tm_mon); time->tm_year = bcd2bin(time->tm_year); #ifdef CONFIG_ACPI p.century = bcd2bin(p.century); #endif } #ifdef CONFIG_MACH_DECSTATION time->tm_year += p.real_year - 72; #endif #ifdef CONFIG_ACPI if (p.century > 19) time->tm_year += (p.century - 19) * 100; #endif /* * Account for differences between how the RTC uses the values * and how they are defined in a struct rtc_time; */ if (time->tm_year <= 69) time->tm_year += 100; time->tm_mon--; return 0; } EXPORT_SYMBOL_GPL(mc146818_get_time); /* AMD systems don't allow access to AltCentury with DV1 */ static bool apply_amd_register_a_behavior(void) { #ifdef CONFIG_X86 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) return true; #endif return false; } /* Set the current date and time in the real time clock. */ int mc146818_set_time(struct rtc_time *time) { unsigned long flags; unsigned char mon, day, hrs, min, sec; unsigned char save_control, save_freq_select; unsigned int yrs; #ifdef CONFIG_MACH_DECSTATION unsigned int real_yrs; #endif unsigned char century = 0; yrs = time->tm_year; mon = time->tm_mon + 1; /* tm_mon starts at zero */ day = time->tm_mday; hrs = time->tm_hour; min = time->tm_min; sec = time->tm_sec; if (yrs > 255) /* They are unsigned */ return -EINVAL; #ifdef CONFIG_MACH_DECSTATION real_yrs = yrs; yrs = 72; /* * We want to keep the year set to 73 until March * for non-leap years, so that Feb, 29th is handled * correctly. */ if (!is_leap_year(real_yrs + 1900) && mon < 3) { real_yrs--; yrs = 73; } #endif #ifdef CONFIG_ACPI if (acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID && acpi_gbl_FADT.century) { century = (yrs + 1900) / 100; yrs %= 100; } #endif /* These limits and adjustments are independent of * whether the chip is in binary mode or not. */ if (yrs > 169) return -EINVAL; if (yrs >= 100) yrs -= 100; spin_lock_irqsave(&rtc_lock, flags); save_control = CMOS_READ(RTC_CONTROL); spin_unlock_irqrestore(&rtc_lock, flags); if (!(save_control & RTC_DM_BINARY) || RTC_ALWAYS_BCD) { sec = bin2bcd(sec); min = bin2bcd(min); hrs = bin2bcd(hrs); day = bin2bcd(day); mon = bin2bcd(mon); yrs = bin2bcd(yrs); century = bin2bcd(century); } spin_lock_irqsave(&rtc_lock, flags); save_control = CMOS_READ(RTC_CONTROL); CMOS_WRITE((save_control|RTC_SET), RTC_CONTROL); save_freq_select = CMOS_READ(RTC_FREQ_SELECT); if (apply_amd_register_a_behavior()) CMOS_WRITE((save_freq_select & ~RTC_AMD_BANK_SELECT), RTC_FREQ_SELECT); else CMOS_WRITE((save_freq_select|RTC_DIV_RESET2), RTC_FREQ_SELECT); #ifdef CONFIG_MACH_DECSTATION CMOS_WRITE(real_yrs, RTC_DEC_YEAR); #endif CMOS_WRITE(yrs, RTC_YEAR); CMOS_WRITE(mon, RTC_MONTH); CMOS_WRITE(day, RTC_DAY_OF_MONTH); CMOS_WRITE(hrs, RTC_HOURS); CMOS_WRITE(min, RTC_MINUTES); CMOS_WRITE(sec, RTC_SECONDS); #ifdef CONFIG_ACPI if (acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID && acpi_gbl_FADT.century) CMOS_WRITE(century, acpi_gbl_FADT.century); #endif CMOS_WRITE(save_control, RTC_CONTROL); CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT); spin_unlock_irqrestore(&rtc_lock, flags); return 0; } EXPORT_SYMBOL_GPL(mc146818_set_time); |
| 18 13 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 | // SPDX-License-Identifier: GPL-2.0 /* * The Virtual DTV test driver serves as a reference DVB driver and helps * validate the existing APIs in the media subsystem. It can also aid * developers working on userspace applications. * * When this module is loaded, it will attempt to modprobe 'dvb_vidtv_tuner' * and 'dvb_vidtv_demod'. * * Copyright (C) 2020 Daniel W. S. Almeida */ #include <linux/dev_printk.h> #include <linux/moduleparam.h> #include <linux/mutex.h> #include <linux/platform_device.h> #include <linux/time.h> #include <linux/types.h> #include <linux/workqueue.h> #include <media/dvbdev.h> #include <media/media-device.h> #include "vidtv_bridge.h" #include "vidtv_common.h" #include "vidtv_demod.h" #include "vidtv_mux.h" #include "vidtv_ts.h" #include "vidtv_tuner.h" #define MUX_BUF_MIN_SZ 90164 #define MUX_BUF_MAX_SZ (MUX_BUF_MIN_SZ * 10) #define TUNER_DEFAULT_ADDR 0x68 #define DEMOD_DEFAULT_ADDR 0x60 #define VIDTV_DEFAULT_NETWORK_ID 0xff44 #define VIDTV_DEFAULT_NETWORK_NAME "LinuxTV.org" #define VIDTV_DEFAULT_TS_ID 0x4081 /* * The LNBf fake parameters here are the ranges used by an * Universal (extended) European LNBf, which is likely the most common LNBf * found on Satellite digital TV system nowadays. */ #define LNB_CUT_FREQUENCY 11700000 /* high IF frequency */ #define LNB_LOW_FREQ 9750000 /* low IF frequency */ #define LNB_HIGH_FREQ 10600000 /* transition frequency */ static unsigned int drop_tslock_prob_on_low_snr; module_param(drop_tslock_prob_on_low_snr, uint, 0444); MODULE_PARM_DESC(drop_tslock_prob_on_low_snr, "Probability of losing the TS lock if the signal quality is bad"); static unsigned int recover_tslock_prob_on_good_snr; module_param(recover_tslock_prob_on_good_snr, uint, 0444); MODULE_PARM_DESC(recover_tslock_prob_on_good_snr, "Probability recovering the TS lock when the signal improves"); static unsigned int mock_power_up_delay_msec; module_param(mock_power_up_delay_msec, uint, 0444); MODULE_PARM_DESC(mock_power_up_delay_msec, "Simulate a power up delay"); static unsigned int mock_tune_delay_msec; module_param(mock_tune_delay_msec, uint, 0444); MODULE_PARM_DESC(mock_tune_delay_msec, "Simulate a tune delay"); static unsigned int vidtv_valid_dvb_t_freqs[NUM_VALID_TUNER_FREQS] = { 474000000 }; module_param_array(vidtv_valid_dvb_t_freqs, uint, NULL, 0444); MODULE_PARM_DESC(vidtv_valid_dvb_t_freqs, "Valid DVB-T frequencies to simulate, in Hz"); static unsigned int vidtv_valid_dvb_c_freqs[NUM_VALID_TUNER_FREQS] = { 474000000 }; module_param_array(vidtv_valid_dvb_c_freqs, uint, NULL, 0444); MODULE_PARM_DESC(vidtv_valid_dvb_c_freqs, "Valid DVB-C frequencies to simulate, in Hz"); static unsigned int vidtv_valid_dvb_s_freqs[NUM_VALID_TUNER_FREQS] = { 11362000 }; module_param_array(vidtv_valid_dvb_s_freqs, uint, NULL, 0444); MODULE_PARM_DESC(vidtv_valid_dvb_s_freqs, "Valid DVB-S/S2 frequencies to simulate at Ku-Band, in kHz"); static unsigned int max_frequency_shift_hz; module_param(max_frequency_shift_hz, uint, 0444); MODULE_PARM_DESC(max_frequency_shift_hz, "Maximum shift in HZ allowed when tuning in a channel"); DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nums); /* * Influences the signal acquisition time. See ISO/IEC 13818-1 : 2000. p. 113. */ static unsigned int si_period_msec = 40; module_param(si_period_msec, uint, 0444); MODULE_PARM_DESC(si_period_msec, "How often to send SI packets. Default: 40ms"); static unsigned int pcr_period_msec = 40; module_param(pcr_period_msec, uint, 0444); MODULE_PARM_DESC(pcr_period_msec, "How often to send PCR packets. Default: 40ms"); static unsigned int mux_rate_kbytes_sec = 4096; module_param(mux_rate_kbytes_sec, uint, 0444); MODULE_PARM_DESC(mux_rate_kbytes_sec, "Mux rate: will pad stream if below"); static unsigned int pcr_pid = 0x200; module_param(pcr_pid, uint, 0444); MODULE_PARM_DESC(pcr_pid, "PCR PID for all channels: defaults to 0x200"); static unsigned int mux_buf_sz_pkts; module_param(mux_buf_sz_pkts, uint, 0444); MODULE_PARM_DESC(mux_buf_sz_pkts, "Size for the internal mux buffer in multiples of 188 bytes"); static u32 vidtv_bridge_mux_buf_sz_for_mux_rate(void) { u32 max_elapsed_time_msecs = VIDTV_MAX_SLEEP_USECS / USEC_PER_MSEC; u32 mux_buf_sz = mux_buf_sz_pkts * TS_PACKET_LEN; u32 nbytes_expected; nbytes_expected = mux_rate_kbytes_sec; nbytes_expected *= max_elapsed_time_msecs; mux_buf_sz = roundup(nbytes_expected, TS_PACKET_LEN); mux_buf_sz += mux_buf_sz / 10; if (mux_buf_sz < MUX_BUF_MIN_SZ) mux_buf_sz = MUX_BUF_MIN_SZ; if (mux_buf_sz > MUX_BUF_MAX_SZ) mux_buf_sz = MUX_BUF_MAX_SZ; return mux_buf_sz; } static bool vidtv_bridge_check_demod_lock(struct vidtv_dvb *dvb, u32 n) { enum fe_status status; dvb->fe[n]->ops.read_status(dvb->fe[n], &status); return status == (FE_HAS_SIGNAL | FE_HAS_CARRIER | FE_HAS_VITERBI | FE_HAS_SYNC | FE_HAS_LOCK); } /* * called on a separate thread by the mux when new packets become available */ static void vidtv_bridge_on_new_pkts_avail(void *priv, u8 *buf, u32 npkts) { struct vidtv_dvb *dvb = priv; /* drop packets if we lose the lock */ if (vidtv_bridge_check_demod_lock(dvb, 0)) dvb_dmx_swfilter_packets(&dvb->demux, buf, npkts); } static int vidtv_start_streaming(struct vidtv_dvb *dvb) { struct vidtv_mux_init_args mux_args = { .mux_rate_kbytes_sec = mux_rate_kbytes_sec, .on_new_packets_available_cb = vidtv_bridge_on_new_pkts_avail, .pcr_period_usecs = pcr_period_msec * USEC_PER_MSEC, .si_period_usecs = si_period_msec * USEC_PER_MSEC, .pcr_pid = pcr_pid, .transport_stream_id = VIDTV_DEFAULT_TS_ID, .network_id = VIDTV_DEFAULT_NETWORK_ID, .network_name = VIDTV_DEFAULT_NETWORK_NAME, .priv = dvb, }; struct device *dev = &dvb->pdev->dev; u32 mux_buf_sz; if (dvb->streaming) { dev_warn_ratelimited(dev, "Already streaming. Skipping.\n"); return 0; } if (mux_buf_sz_pkts) mux_buf_sz = mux_buf_sz_pkts; else mux_buf_sz = vidtv_bridge_mux_buf_sz_for_mux_rate(); mux_args.mux_buf_sz = mux_buf_sz; dvb->mux = vidtv_mux_init(dvb->fe[0], dev, &mux_args); if (!dvb->mux) return -ENOMEM; dvb->streaming = true; vidtv_mux_start_thread(dvb->mux); dev_dbg_ratelimited(dev, "Started streaming\n"); return 0; } static int vidtv_stop_streaming(struct vidtv_dvb *dvb) { struct device *dev = &dvb->pdev->dev; if (!dvb->streaming) { dev_warn_ratelimited(dev, "No streaming. Skipping.\n"); return 0; } dvb->streaming = false; vidtv_mux_stop_thread(dvb->mux); vidtv_mux_destroy(dvb->mux); dvb->mux = NULL; dev_dbg_ratelimited(dev, "Stopped streaming\n"); return 0; } static int vidtv_start_feed(struct dvb_demux_feed *feed) { struct dvb_demux *demux = feed->demux; struct vidtv_dvb *dvb = demux->priv; int ret; int rc; if (!demux->dmx.frontend) return -EINVAL; mutex_lock(&dvb->feed_lock); dvb->nfeeds++; rc = dvb->nfeeds; if (dvb->nfeeds == 1) { ret = vidtv_start_streaming(dvb); if (ret < 0) rc = ret; } mutex_unlock(&dvb->feed_lock); return rc; } static int vidtv_stop_feed(struct dvb_demux_feed *feed) { struct dvb_demux *demux = feed->demux; struct vidtv_dvb *dvb = demux->priv; int err = 0; mutex_lock(&dvb->feed_lock); dvb->nfeeds--; if (!dvb->nfeeds) err = vidtv_stop_streaming(dvb); mutex_unlock(&dvb->feed_lock); return err; } static struct dvb_frontend *vidtv_get_frontend_ptr(struct i2c_client *c) { struct vidtv_demod_state *state = i2c_get_clientdata(c); /* the demod will set this when its probe function runs */ return &state->frontend; } static int vidtv_master_xfer(struct i2c_adapter *i2c_adap, struct i2c_msg msgs[], int num) { /* * Right now, this virtual driver doesn't really send or receive * messages from I2C. A real driver will require an implementation * here. */ return 0; } static u32 vidtv_i2c_func(struct i2c_adapter *adapter) { return I2C_FUNC_I2C; } static const struct i2c_algorithm vidtv_i2c_algorithm = { .master_xfer = vidtv_master_xfer, .functionality = vidtv_i2c_func, }; static int vidtv_bridge_i2c_register_adap(struct vidtv_dvb *dvb) { struct i2c_adapter *i2c_adapter = &dvb->i2c_adapter; strscpy(i2c_adapter->name, "vidtv_i2c", sizeof(i2c_adapter->name)); i2c_adapter->owner = THIS_MODULE; i2c_adapter->algo = &vidtv_i2c_algorithm; i2c_adapter->algo_data = NULL; i2c_adapter->timeout = 500; i2c_adapter->retries = 3; i2c_adapter->dev.parent = &dvb->pdev->dev; i2c_set_adapdata(i2c_adapter, dvb); return i2c_add_adapter(&dvb->i2c_adapter); } static int vidtv_bridge_register_adap(struct vidtv_dvb *dvb) { int ret = 0; ret = dvb_register_adapter(&dvb->adapter, KBUILD_MODNAME, THIS_MODULE, &dvb->i2c_adapter.dev, adapter_nums); return ret; } static int vidtv_bridge_dmx_init(struct vidtv_dvb *dvb) { dvb->demux.dmx.capabilities = DMX_TS_FILTERING | DMX_SECTION_FILTERING; dvb->demux.priv = dvb; dvb->demux.filternum = 256; dvb->demux.feednum = 256; dvb->demux.start_feed = vidtv_start_feed; dvb->demux.stop_feed = vidtv_stop_feed; return dvb_dmx_init(&dvb->demux); } static int vidtv_bridge_dmxdev_init(struct vidtv_dvb *dvb) { dvb->dmx_dev.filternum = 256; dvb->dmx_dev.demux = &dvb->demux.dmx; dvb->dmx_dev.capabilities = 0; return dvb_dmxdev_init(&dvb->dmx_dev, &dvb->adapter); } static int vidtv_bridge_probe_demod(struct vidtv_dvb *dvb, u32 n) { struct vidtv_demod_config cfg = { .drop_tslock_prob_on_low_snr = drop_tslock_prob_on_low_snr, .recover_tslock_prob_on_good_snr = recover_tslock_prob_on_good_snr, }; dvb->i2c_client_demod[n] = dvb_module_probe("dvb_vidtv_demod", NULL, &dvb->i2c_adapter, DEMOD_DEFAULT_ADDR, &cfg); /* driver will not work anyways so bail out */ if (!dvb->i2c_client_demod[n]) return -ENODEV; /* retrieve a ptr to the frontend state */ dvb->fe[n] = vidtv_get_frontend_ptr(dvb->i2c_client_demod[n]); return 0; } static int vidtv_bridge_probe_tuner(struct vidtv_dvb *dvb, u32 n) { struct vidtv_tuner_config cfg = { .fe = dvb->fe[n], .mock_power_up_delay_msec = mock_power_up_delay_msec, .mock_tune_delay_msec = mock_tune_delay_msec, }; u32 freq; int i; /* TODO: check if the frequencies are at a valid range */ memcpy(cfg.vidtv_valid_dvb_t_freqs, vidtv_valid_dvb_t_freqs, sizeof(vidtv_valid_dvb_t_freqs)); memcpy(cfg.vidtv_valid_dvb_c_freqs, vidtv_valid_dvb_c_freqs, sizeof(vidtv_valid_dvb_c_freqs)); /* * Convert Satellite frequencies from Ku-band in kHZ into S-band * frequencies in Hz. */ for (i = 0; i < ARRAY_SIZE(vidtv_valid_dvb_s_freqs); i++) { freq = vidtv_valid_dvb_s_freqs[i]; if (freq) { if (freq < LNB_CUT_FREQUENCY) freq = abs(freq - LNB_LOW_FREQ); else freq = abs(freq - LNB_HIGH_FREQ); } cfg.vidtv_valid_dvb_s_freqs[i] = freq; } cfg.max_frequency_shift_hz = max_frequency_shift_hz; dvb->i2c_client_tuner[n] = dvb_module_probe("dvb_vidtv_tuner", NULL, &dvb->i2c_adapter, TUNER_DEFAULT_ADDR, &cfg); return (dvb->i2c_client_tuner[n]) ? 0 : -ENODEV; } static int vidtv_bridge_dvb_init(struct vidtv_dvb *dvb) { int ret, i, j; ret = vidtv_bridge_i2c_register_adap(dvb); if (ret < 0) goto fail_i2c; ret = vidtv_bridge_register_adap(dvb); if (ret < 0) goto fail_adapter; dvb_register_media_controller(&dvb->adapter, &dvb->mdev); for (i = 0; i < NUM_FE; ++i) { ret = vidtv_bridge_probe_demod(dvb, i); if (ret < 0) goto fail_demod_probe; ret = vidtv_bridge_probe_tuner(dvb, i); if (ret < 0) goto fail_tuner_probe; ret = dvb_register_frontend(&dvb->adapter, dvb->fe[i]); if (ret < 0) goto fail_fe; } ret = vidtv_bridge_dmx_init(dvb); if (ret < 0) goto fail_dmx; ret = vidtv_bridge_dmxdev_init(dvb); if (ret < 0) goto fail_dmx_dev; for (j = 0; j < NUM_FE; ++j) { ret = dvb->demux.dmx.connect_frontend(&dvb->demux.dmx, &dvb->dmx_fe[j]); if (ret < 0) goto fail_dmx_conn; /* * The source of the demux is a frontend connected * to the demux. */ dvb->dmx_fe[j].source = DMX_FRONTEND_0; } return ret; fail_dmx_conn: for (j = j - 1; j >= 0; --j) dvb->demux.dmx.remove_frontend(&dvb->demux.dmx, &dvb->dmx_fe[j]); dvb_dmxdev_release(&dvb->dmx_dev); fail_dmx_dev: dvb_dmx_release(&dvb->demux); fail_dmx: fail_demod_probe: for (i = i - 1; i >= 0; --i) { dvb_unregister_frontend(dvb->fe[i]); fail_fe: dvb_module_release(dvb->i2c_client_tuner[i]); fail_tuner_probe: dvb_module_release(dvb->i2c_client_demod[i]); } fail_adapter: dvb_unregister_adapter(&dvb->adapter); fail_i2c: i2c_del_adapter(&dvb->i2c_adapter); return ret; } static int vidtv_bridge_probe(struct platform_device *pdev) { struct vidtv_dvb *dvb; int ret; dvb = kzalloc(sizeof(*dvb), GFP_KERNEL); if (!dvb) return -ENOMEM; dvb->pdev = pdev; #ifdef CONFIG_MEDIA_CONTROLLER_DVB dvb->mdev.dev = &pdev->dev; strscpy(dvb->mdev.model, "vidtv", sizeof(dvb->mdev.model)); strscpy(dvb->mdev.bus_info, "platform:vidtv", sizeof(dvb->mdev.bus_info)); media_device_init(&dvb->mdev); #endif ret = vidtv_bridge_dvb_init(dvb); if (ret < 0) goto err_dvb; mutex_init(&dvb->feed_lock); platform_set_drvdata(pdev, dvb); #ifdef CONFIG_MEDIA_CONTROLLER_DVB ret = media_device_register(&dvb->mdev); if (ret) { dev_err(dvb->mdev.dev, "media device register failed (err=%d)\n", ret); goto err_media_device_register; } #endif /* CONFIG_MEDIA_CONTROLLER_DVB */ dev_info(&pdev->dev, "Successfully initialized vidtv!\n"); return ret; #ifdef CONFIG_MEDIA_CONTROLLER_DVB err_media_device_register: media_device_cleanup(&dvb->mdev); #endif /* CONFIG_MEDIA_CONTROLLER_DVB */ err_dvb: kfree(dvb); return ret; } static void vidtv_bridge_remove(struct platform_device *pdev) { struct vidtv_dvb *dvb; u32 i; dvb = platform_get_drvdata(pdev); #ifdef CONFIG_MEDIA_CONTROLLER_DVB media_device_unregister(&dvb->mdev); media_device_cleanup(&dvb->mdev); #endif /* CONFIG_MEDIA_CONTROLLER_DVB */ mutex_destroy(&dvb->feed_lock); for (i = 0; i < NUM_FE; ++i) { dvb_unregister_frontend(dvb->fe[i]); dvb_module_release(dvb->i2c_client_tuner[i]); dvb_module_release(dvb->i2c_client_demod[i]); } dvb_dmxdev_release(&dvb->dmx_dev); dvb_dmx_release(&dvb->demux); dvb_unregister_adapter(&dvb->adapter); dev_info(&pdev->dev, "Successfully removed vidtv\n"); } static void vidtv_bridge_dev_release(struct device *dev) { struct vidtv_dvb *dvb; dvb = dev_get_drvdata(dev); kfree(dvb); } static struct platform_device vidtv_bridge_dev = { .name = VIDTV_PDEV_NAME, .dev.release = vidtv_bridge_dev_release, }; static struct platform_driver vidtv_bridge_driver = { .driver = { .name = VIDTV_PDEV_NAME, }, .probe = vidtv_bridge_probe, .remove = vidtv_bridge_remove, }; static void __exit vidtv_bridge_exit(void) { platform_driver_unregister(&vidtv_bridge_driver); platform_device_unregister(&vidtv_bridge_dev); } static int __init vidtv_bridge_init(void) { int ret; ret = platform_device_register(&vidtv_bridge_dev); if (ret) return ret; ret = platform_driver_register(&vidtv_bridge_driver); if (ret) platform_device_unregister(&vidtv_bridge_dev); return ret; } module_init(vidtv_bridge_init); module_exit(vidtv_bridge_exit); MODULE_DESCRIPTION("Virtual Digital TV Test Driver"); MODULE_AUTHOR("Daniel W. S. Almeida"); MODULE_LICENSE("GPL"); MODULE_ALIAS("vidtv"); MODULE_ALIAS("dvb_vidtv"); |
| 5 5 5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 | // SPDX-License-Identifier: GPL-2.0 #include <linux/ceph/ceph_debug.h> #include <linux/device.h> #include <linux/slab.h> #include <linux/module.h> #include <linux/ctype.h> #include <linux/debugfs.h> #include <linux/seq_file.h> #include <linux/math64.h> #include <linux/ktime.h> #include <linux/ceph/libceph.h> #include <linux/ceph/mon_client.h> #include <linux/ceph/auth.h> #include <linux/ceph/debugfs.h> #include "super.h" #ifdef CONFIG_DEBUG_FS #include "mds_client.h" #include "metric.h" static int mdsmap_show(struct seq_file *s, void *p) { int i; struct ceph_fs_client *fsc = s->private; struct ceph_mdsmap *mdsmap; if (!fsc->mdsc || !fsc->mdsc->mdsmap) return 0; mdsmap = fsc->mdsc->mdsmap; seq_printf(s, "epoch %d\n", mdsmap->m_epoch); seq_printf(s, "root %d\n", mdsmap->m_root); seq_printf(s, "max_mds %d\n", mdsmap->m_max_mds); seq_printf(s, "session_timeout %d\n", mdsmap->m_session_timeout); seq_printf(s, "session_autoclose %d\n", mdsmap->m_session_autoclose); for (i = 0; i < mdsmap->possible_max_rank; i++) { struct ceph_entity_addr *addr = &mdsmap->m_info[i].addr; int state = mdsmap->m_info[i].state; seq_printf(s, "\tmds%d\t%s\t(%s)\n", i, ceph_pr_addr(addr), ceph_mds_state_name(state)); } return 0; } /* * mdsc debugfs */ static int mdsc_show(struct seq_file *s, void *p) { struct ceph_fs_client *fsc = s->private; struct ceph_mds_client *mdsc = fsc->mdsc; struct ceph_mds_request *req; struct rb_node *rp; char *path; mutex_lock(&mdsc->mutex); for (rp = rb_first(&mdsc->request_tree); rp; rp = rb_next(rp)) { req = rb_entry(rp, struct ceph_mds_request, r_node); if (req->r_request && req->r_session) seq_printf(s, "%lld\tmds%d\t", req->r_tid, req->r_session->s_mds); else if (!req->r_request) seq_printf(s, "%lld\t(no request)\t", req->r_tid); else seq_printf(s, "%lld\t(no session)\t", req->r_tid); seq_printf(s, "%s", ceph_mds_op_name(req->r_op)); if (test_bit(CEPH_MDS_R_GOT_UNSAFE, &req->r_req_flags)) seq_puts(s, "\t(unsafe)"); else seq_puts(s, "\t"); if (req->r_inode) { seq_printf(s, " #%llx", ceph_ino(req->r_inode)); } else if (req->r_dentry) { struct ceph_path_info path_info; path = ceph_mdsc_build_path(mdsc, req->r_dentry, &path_info, 0); if (IS_ERR(path)) path = NULL; spin_lock(&req->r_dentry->d_lock); seq_printf(s, " #%llx/%pd (%s)", ceph_ino(d_inode(req->r_dentry->d_parent)), req->r_dentry, path ? path : ""); spin_unlock(&req->r_dentry->d_lock); ceph_mdsc_free_path_info(&path_info); } else if (req->r_path1) { seq_printf(s, " #%llx/%s", req->r_ino1.ino, req->r_path1); } else { seq_printf(s, " #%llx", req->r_ino1.ino); } if (req->r_old_dentry) { struct ceph_path_info path_info; path = ceph_mdsc_build_path(mdsc, req->r_old_dentry, &path_info, 0); if (IS_ERR(path)) path = NULL; spin_lock(&req->r_old_dentry->d_lock); seq_printf(s, " #%llx/%pd (%s)", req->r_old_dentry_dir ? ceph_ino(req->r_old_dentry_dir) : 0, req->r_old_dentry, path ? path : ""); spin_unlock(&req->r_old_dentry->d_lock); ceph_mdsc_free_path_info(&path_info); } else if (req->r_path2 && req->r_op != CEPH_MDS_OP_SYMLINK) { if (req->r_ino2.ino) seq_printf(s, " #%llx/%s", req->r_ino2.ino, req->r_path2); else seq_printf(s, " %s", req->r_path2); } seq_puts(s, "\n"); } mutex_unlock(&mdsc->mutex); return 0; } #define CEPH_LAT_METRIC_SHOW(name, total, avg, min, max, sq) { \ s64 _total, _avg, _min, _max, _sq, _st; \ _avg = ktime_to_us(avg); \ _min = ktime_to_us(min == KTIME_MAX ? 0 : min); \ _max = ktime_to_us(max); \ _total = total - 1; \ _sq = _total > 0 ? DIV64_U64_ROUND_CLOSEST(sq, _total) : 0; \ _st = int_sqrt64(_sq); \ _st = ktime_to_us(_st); \ seq_printf(s, "%-14s%-12lld%-16lld%-16lld%-16lld%lld\n", \ name, total, _avg, _min, _max, _st); \ } #define CEPH_SZ_METRIC_SHOW(name, total, avg, min, max, sum) { \ u64 _min = min == U64_MAX ? 0 : min; \ seq_printf(s, "%-14s%-12lld%-16llu%-16llu%-16llu%llu\n", \ name, total, avg, _min, max, sum); \ } static int metrics_file_show(struct seq_file *s, void *p) { struct ceph_fs_client *fsc = s->private; struct ceph_client_metric *m = &fsc->mdsc->metric; seq_printf(s, "item total\n"); seq_printf(s, "------------------------------------------\n"); seq_printf(s, "%-35s%lld\n", "total inodes", percpu_counter_sum(&m->total_inodes)); seq_printf(s, "%-35s%lld\n", "opened files", atomic64_read(&m->opened_files)); seq_printf(s, "%-35s%lld\n", "pinned i_caps", atomic64_read(&m->total_caps)); seq_printf(s, "%-35s%lld\n", "opened inodes", percpu_counter_sum(&m->opened_inodes)); return 0; } static const char * const metric_str[] = { "read", "write", "metadata", "copyfrom" }; static int metrics_latency_show(struct seq_file *s, void *p) { struct ceph_fs_client *fsc = s->private; struct ceph_client_metric *cm = &fsc->mdsc->metric; struct ceph_metric *m; s64 total, avg, min, max, sq; int i; seq_printf(s, "item total avg_lat(us) min_lat(us) max_lat(us) stdev(us)\n"); seq_printf(s, "-----------------------------------------------------------------------------------\n"); for (i = 0; i < METRIC_MAX; i++) { m = &cm->metric[i]; spin_lock(&m->lock); total = m->total; avg = m->latency_avg; min = m->latency_min; max = m->latency_max; sq = m->latency_sq_sum; spin_unlock(&m->lock); CEPH_LAT_METRIC_SHOW(metric_str[i], total, avg, min, max, sq); } return 0; } static int metrics_size_show(struct seq_file *s, void *p) { struct ceph_fs_client *fsc = s->private; struct ceph_client_metric *cm = &fsc->mdsc->metric; struct ceph_metric *m; s64 total; u64 sum, avg, min, max; int i; seq_printf(s, "item total avg_sz(bytes) min_sz(bytes) max_sz(bytes) total_sz(bytes)\n"); seq_printf(s, "----------------------------------------------------------------------------------------\n"); for (i = 0; i < METRIC_MAX; i++) { /* skip 'metadata' as it doesn't use the size metric */ if (i == METRIC_METADATA) continue; m = &cm->metric[i]; spin_lock(&m->lock); total = m->total; sum = m->size_sum; avg = total > 0 ? DIV64_U64_ROUND_CLOSEST(sum, total) : 0; min = m->size_min; max = m->size_max; spin_unlock(&m->lock); CEPH_SZ_METRIC_SHOW(metric_str[i], total, avg, min, max, sum); } return 0; } static int metrics_caps_show(struct seq_file *s, void *p) { struct ceph_fs_client *fsc = s->private; struct ceph_client_metric *m = &fsc->mdsc->metric; int nr_caps = 0; seq_printf(s, "item total miss hit\n"); seq_printf(s, "-------------------------------------------------\n"); seq_printf(s, "%-14s%-16lld%-16lld%lld\n", "d_lease", atomic64_read(&m->total_dentries), percpu_counter_sum(&m->d_lease_mis), percpu_counter_sum(&m->d_lease_hit)); nr_caps = atomic64_read(&m->total_caps); seq_printf(s, "%-14s%-16d%-16lld%lld\n", "caps", nr_caps, percpu_counter_sum(&m->i_caps_mis), percpu_counter_sum(&m->i_caps_hit)); return 0; } static int caps_show_cb(struct inode *inode, int mds, void *p) { struct ceph_inode_info *ci = ceph_inode(inode); struct seq_file *s = p; struct ceph_cap *cap; spin_lock(&ci->i_ceph_lock); cap = __get_cap_for_mds(ci, mds); if (cap) seq_printf(s, "0x%-17llx%-3d%-17s%-17s\n", ceph_ino(inode), cap->session->s_mds, ceph_cap_string(cap->issued), ceph_cap_string(cap->implemented)); spin_unlock(&ci->i_ceph_lock); return 0; } static int caps_show(struct seq_file *s, void *p) { struct ceph_fs_client *fsc = s->private; struct ceph_mds_client *mdsc = fsc->mdsc; int total, avail, used, reserved, min, i; struct cap_wait *cw; ceph_reservation_status(fsc, &total, &avail, &used, &reserved, &min); seq_printf(s, "total\t\t%d\n" "avail\t\t%d\n" "used\t\t%d\n" "reserved\t%d\n" "min\t\t%d\n\n", total, avail, used, reserved, min); seq_printf(s, "ino mds issued implemented\n"); seq_printf(s, "--------------------------------------------------\n"); mutex_lock(&mdsc->mutex); for (i = 0; i < mdsc->max_sessions; i++) { struct ceph_mds_session *session; session = __ceph_lookup_mds_session(mdsc, i); if (!session) continue; mutex_unlock(&mdsc->mutex); mutex_lock(&session->s_mutex); ceph_iterate_session_caps(session, caps_show_cb, s); mutex_unlock(&session->s_mutex); ceph_put_mds_session(session); mutex_lock(&mdsc->mutex); } mutex_unlock(&mdsc->mutex); seq_printf(s, "\n\nWaiters:\n--------\n"); seq_printf(s, "tgid ino need want\n"); seq_printf(s, "-----------------------------------------------------\n"); spin_lock(&mdsc->caps_list_lock); list_for_each_entry(cw, &mdsc->cap_wait_list, list) { seq_printf(s, "%-13d0x%-17llx%-17s%-17s\n", cw->tgid, cw->ino, ceph_cap_string(cw->need), ceph_cap_string(cw->want)); } spin_unlock(&mdsc->caps_list_lock); return 0; } static int mds_sessions_show(struct seq_file *s, void *ptr) { struct ceph_fs_client *fsc = s->private; struct ceph_mds_client *mdsc = fsc->mdsc; struct ceph_auth_client *ac = fsc->client->monc.auth; struct ceph_options *opt = fsc->client->options; int mds; mutex_lock(&mdsc->mutex); /* The 'num' portion of an 'entity name' */ seq_printf(s, "global_id %llu\n", ac->global_id); /* The -o name mount argument */ seq_printf(s, "name \"%s\"\n", opt->name ? opt->name : ""); /* The list of MDS session rank+state */ for (mds = 0; mds < mdsc->max_sessions; mds++) { struct ceph_mds_session *session = __ceph_lookup_mds_session(mdsc, mds); if (!session) { continue; } mutex_unlock(&mdsc->mutex); seq_printf(s, "mds.%d %s\n", session->s_mds, ceph_session_state_name(session->s_state)); ceph_put_mds_session(session); mutex_lock(&mdsc->mutex); } mutex_unlock(&mdsc->mutex); return 0; } static int status_show(struct seq_file *s, void *p) { struct ceph_fs_client *fsc = s->private; struct ceph_entity_inst *inst = &fsc->client->msgr.inst; struct ceph_entity_addr *client_addr = ceph_client_addr(fsc->client); seq_printf(s, "instance: %s.%lld %s/%u\n", ENTITY_NAME(inst->name), ceph_pr_addr(client_addr), le32_to_cpu(client_addr->nonce)); seq_printf(s, "blocklisted: %s\n", str_true_false(fsc->blocklisted)); return 0; } DEFINE_SHOW_ATTRIBUTE(mdsmap); DEFINE_SHOW_ATTRIBUTE(mdsc); DEFINE_SHOW_ATTRIBUTE(caps); DEFINE_SHOW_ATTRIBUTE(mds_sessions); DEFINE_SHOW_ATTRIBUTE(status); DEFINE_SHOW_ATTRIBUTE(metrics_file); DEFINE_SHOW_ATTRIBUTE(metrics_latency); DEFINE_SHOW_ATTRIBUTE(metrics_size); DEFINE_SHOW_ATTRIBUTE(metrics_caps); /* * debugfs */ static int congestion_kb_set(void *data, u64 val) { struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; fsc->mount_options->congestion_kb = (int)val; return 0; } static int congestion_kb_get(void *data, u64 *val) { struct ceph_fs_client *fsc = (struct ceph_fs_client *)data; *val = (u64)fsc->mount_options->congestion_kb; return 0; } DEFINE_SIMPLE_ATTRIBUTE(congestion_kb_fops, congestion_kb_get, congestion_kb_set, "%llu\n"); void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc) { doutc(fsc->client, "begin\n"); debugfs_remove(fsc->debugfs_bdi); debugfs_remove(fsc->debugfs_congestion_kb); debugfs_remove(fsc->debugfs_mdsmap); debugfs_remove(fsc->debugfs_mds_sessions); debugfs_remove(fsc->debugfs_caps); debugfs_remove(fsc->debugfs_status); debugfs_remove(fsc->debugfs_mdsc); debugfs_remove_recursive(fsc->debugfs_metrics_dir); doutc(fsc->client, "done\n"); } void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) { char name[NAME_MAX]; doutc(fsc->client, "begin\n"); fsc->debugfs_congestion_kb = debugfs_create_file("writeback_congestion_kb", 0600, fsc->client->debugfs_dir, fsc, &congestion_kb_fops); snprintf(name, sizeof(name), "../../bdi/%s", bdi_dev_name(fsc->sb->s_bdi)); fsc->debugfs_bdi = debugfs_create_symlink("bdi", fsc->client->debugfs_dir, name); fsc->debugfs_mdsmap = debugfs_create_file("mdsmap", 0400, fsc->client->debugfs_dir, fsc, &mdsmap_fops); fsc->debugfs_mds_sessions = debugfs_create_file("mds_sessions", 0400, fsc->client->debugfs_dir, fsc, &mds_sessions_fops); fsc->debugfs_mdsc = debugfs_create_file("mdsc", 0400, fsc->client->debugfs_dir, fsc, &mdsc_fops); fsc->debugfs_caps = debugfs_create_file("caps", 0400, fsc->client->debugfs_dir, fsc, &caps_fops); fsc->debugfs_status = debugfs_create_file("status", 0400, fsc->client->debugfs_dir, fsc, &status_fops); fsc->debugfs_metrics_dir = debugfs_create_dir("metrics", fsc->client->debugfs_dir); debugfs_create_file("file", 0400, fsc->debugfs_metrics_dir, fsc, &metrics_file_fops); debugfs_create_file("latency", 0400, fsc->debugfs_metrics_dir, fsc, &metrics_latency_fops); debugfs_create_file("size", 0400, fsc->debugfs_metrics_dir, fsc, &metrics_size_fops); debugfs_create_file("caps", 0400, fsc->debugfs_metrics_dir, fsc, &metrics_caps_fops); doutc(fsc->client, "done\n"); } #else /* CONFIG_DEBUG_FS */ void ceph_fs_debugfs_init(struct ceph_fs_client *fsc) { } void ceph_fs_debugfs_cleanup(struct ceph_fs_client *fsc) { } #endif /* CONFIG_DEBUG_FS */ |
| 2 2 3 3 3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 | /* * `Soft' font definitions * * Created 1995 by Geert Uytterhoeven * Rewritten 1998 by Martin Mares <mj@ucw.cz> * * 2001 - Documented with DocBook * - Brad Douglas <brad@neruo.com> * * This file is subject to the terms and conditions of the GNU General Public * License. See the file COPYING in the main directory of this archive * for more details. */ #include <linux/module.h> #include <linux/types.h> #include <linux/string.h> #if defined(__mc68000__) #include <asm/setup.h> #endif #include <linux/font.h> static const struct font_desc *fonts[] = { #ifdef CONFIG_FONT_8x8 &font_vga_8x8, #endif #ifdef CONFIG_FONT_8x16 &font_vga_8x16, #endif #ifdef CONFIG_FONT_6x11 &font_vga_6x11, #endif #ifdef CONFIG_FONT_7x14 &font_7x14, #endif #ifdef CONFIG_FONT_SUN8x16 &font_sun_8x16, #endif #ifdef CONFIG_FONT_SUN12x22 &font_sun_12x22, #endif #ifdef CONFIG_FONT_10x18 &font_10x18, #endif #ifdef CONFIG_FONT_ACORN_8x8 &font_acorn_8x8, #endif #ifdef CONFIG_FONT_PEARL_8x8 &font_pearl_8x8, #endif #ifdef CONFIG_FONT_MINI_4x6 &font_mini_4x6, #endif #ifdef CONFIG_FONT_6x10 &font_6x10, #endif #ifdef CONFIG_FONT_TER16x32 &font_ter_16x32, #endif #ifdef CONFIG_FONT_6x8 &font_6x8, #endif }; #define num_fonts ARRAY_SIZE(fonts) #ifdef NO_FONTS #error No fonts configured. #endif /** * find_font - find a font * @name: string name of a font * * Find a specified font with string name @name. * * Returns %NULL if no font found, or a pointer to the * specified font. * */ const struct font_desc *find_font(const char *name) { unsigned int i; BUILD_BUG_ON(!num_fonts); for (i = 0; i < num_fonts; i++) if (!strcmp(fonts[i]->name, name)) return fonts[i]; return NULL; } EXPORT_SYMBOL(find_font); /** * get_default_font - get default font * @xres: screen size of X * @yres: screen size of Y * @font_w: bit array of supported widths (1 - FB_MAX_BLIT_WIDTH) * @font_h: bit array of supported heights (1 - FB_MAX_BLIT_HEIGHT) * * Get the default font for a specified screen size. * Dimensions are in pixels. * * font_w or font_h being NULL means all values are supported. * * Returns %NULL if no font is found, or a pointer to the * chosen font. * */ const struct font_desc *get_default_font(int xres, int yres, unsigned long *font_w, unsigned long *font_h) { int i, c, cc, res; const struct font_desc *f, *g; g = NULL; cc = -10000; for (i = 0; i < num_fonts; i++) { f = fonts[i]; c = f->pref; #if defined(__mc68000__) #ifdef CONFIG_FONT_PEARL_8x8 if (MACH_IS_AMIGA && f->idx == PEARL8x8_IDX) c = 100; #endif #ifdef CONFIG_FONT_6x11 if (MACH_IS_MAC && xres < 640 && f->idx == VGA6x11_IDX) c = 100; #endif #endif if ((yres < 400) == (f->height <= 8)) c += 1000; /* prefer a bigger font for high resolution */ res = (xres / f->width) * (yres / f->height) / 1000; if (res > 20) c += 20 - res; if ((!font_w || test_bit(f->width - 1, font_w)) && (!font_h || test_bit(f->height - 1, font_h))) c += 1000; if (c > cc) { cc = c; g = f; } } return g; } EXPORT_SYMBOL(get_default_font); MODULE_AUTHOR("James Simmons <jsimmons@users.sf.net>"); MODULE_DESCRIPTION("Console Fonts"); MODULE_LICENSE("GPL"); |
| 22 22 148 148 145 148 170 170 63 148 10 6 13 13 6 3 3 3 3 1 3 17 17 17 17 17 16 17 78 46 43 43 40 1 17 7 6 6 3 5 5 5 27 12 18 13 6 6 7 7 6 9 7 9 21 20 29 29 29 21 9 17 17 17 23 23 23 4 23 2 21 17 17 17 17 8 23 8 17 25 25 25 40 14 32 1 22 31 31 9 30 1 1 1 1 22 22 1 22 4 39 37 12 38 37 1 13 24 22 22 22 22 43 43 41 11 21 43 6 4 4 43 11 38 29 11 43 7 43 43 43 18 6 24 23 24 24 24 24 24 24 24 1 1 16 16 24 24 24 24 5 24 24 24 24 15 24 24 24 24 24 6 6 6 6 6 6 5 6 6 6 6 6 6 6 5 24 24 13 13 5 24 24 24 24 24 23 21 5 5 24 41 41 41 9 24 13 24 24 5 37 24 24 39 36 24 1 1 1 1 36 36 2 34 1 10 11 2 19 27 27 11 27 24 69 69 56 56 56 28 29 9 28 27 27 3 27 14 1 27 27 27 1 25 14 24 24 3 14 9 40 9 9 6 1 4 5 2 4 9 9 9 5 3 5 2 4 1 3 3 3 3 25 28 28 3 23 25 23 23 2 16 17 5 1 5 21 21 3 2 17 17 13 4 15 9 9 9 1 1 1 4 1 2 4 1 6 7 6 12 3 1 1 4 17 1 1 1 1 1 10 10 1 10 48 45 5 58 58 58 3 13 49 14 20 43 58 48 14 14 57 17 45 2 46 46 2 10 6 6 10 14 14 14 7 7 7 4 5 6 3 3 3 3 37 37 74 75 37 37 37 37 37 37 59 59 3 18 17 1 21 7 7 16 10 10 6 1 5 55 32 118 19 16 16 16 32 32 32 24 12 12 12 12 12 18 18 18 18 18 18 3 26 56 52 8 8 8 19 19 19 15 4 6 5 14 14 14 17 19 19 18 3 19 3 18 19 95 95 95 58 39 44 44 1 44 44 44 32 32 58 59 59 58 18 18 33 33 1 33 33 33 1 33 378 377 378 38 38 9 38 38 14 35 17 380 34 377 37 32 38 31 58 7 2 2 8 8 8 8 6 2 11 11 11 3 153 2 2 2 2 2 8 8 4 6 9 2 7 6 1 1 20 20 10 1 1 4 5 82 82 82 82 82 82 34 52 32 67 34 52 32 67 28 28 1 14 14 14 19 17 17 17 1 17 17 17 6 6 6 1 5 1 4 4 4 4 4 4 24 30 30 30 30 1 26 4 24 4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 | // SPDX-License-Identifier: GPL-2.0 /* Multipath TCP * * Copyright (c) 2017 - 2019, Intel Corporation. */ #define pr_fmt(fmt) "MPTCP: " fmt #include <linux/kernel.h> #include <linux/module.h> #include <linux/netdevice.h> #include <linux/sched/signal.h> #include <linux/atomic.h> #include <net/aligned_data.h> #include <net/rps.h> #include <net/sock.h> #include <net/inet_common.h> #include <net/inet_hashtables.h> #include <net/protocol.h> #include <net/tcp_states.h> #if IS_ENABLED(CONFIG_MPTCP_IPV6) #include <net/transp_v6.h> #endif #include <net/mptcp.h> #include <net/hotdata.h> #include <net/xfrm.h> #include <asm/ioctls.h> #include "protocol.h" #include "mib.h" #define CREATE_TRACE_POINTS #include <trace/events/mptcp.h> #if IS_ENABLED(CONFIG_MPTCP_IPV6) struct mptcp6_sock { struct mptcp_sock msk; struct ipv6_pinfo np; }; #endif enum { MPTCP_CMSG_TS = BIT(0), MPTCP_CMSG_INQ = BIT(1), }; static struct percpu_counter mptcp_sockets_allocated ____cacheline_aligned_in_smp; static void __mptcp_destroy_sock(struct sock *sk); static void mptcp_check_send_data_fin(struct sock *sk); DEFINE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions) = { .bh_lock = INIT_LOCAL_LOCK(bh_lock), }; static struct net_device *mptcp_napi_dev; /* Returns end sequence number of the receiver's advertised window */ static u64 mptcp_wnd_end(const struct mptcp_sock *msk) { return READ_ONCE(msk->wnd_end); } static const struct proto_ops *mptcp_fallback_tcp_ops(const struct sock *sk) { unsigned short family = READ_ONCE(sk->sk_family); #if IS_ENABLED(CONFIG_MPTCP_IPV6) if (family == AF_INET6) return &inet6_stream_ops; #endif WARN_ON_ONCE(family != AF_INET); return &inet_stream_ops; } bool __mptcp_try_fallback(struct mptcp_sock *msk, int fb_mib) { struct net *net = sock_net((struct sock *)msk); if (__mptcp_check_fallback(msk)) return true; /* The caller possibly is not holding the msk socket lock, but * in the fallback case only the current subflow is touching * the OoO queue. */ if (!RB_EMPTY_ROOT(&msk->out_of_order_queue)) return false; spin_lock_bh(&msk->fallback_lock); if (!msk->allow_infinite_fallback) { spin_unlock_bh(&msk->fallback_lock); return false; } msk->allow_subflows = false; set_bit(MPTCP_FALLBACK_DONE, &msk->flags); __MPTCP_INC_STATS(net, fb_mib); spin_unlock_bh(&msk->fallback_lock); return true; } static int __mptcp_socket_create(struct mptcp_sock *msk) { struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; struct socket *ssock; int err; err = mptcp_subflow_create_socket(sk, sk->sk_family, &ssock); if (err) return err; msk->scaling_ratio = tcp_sk(ssock->sk)->scaling_ratio; WRITE_ONCE(msk->first, ssock->sk); subflow = mptcp_subflow_ctx(ssock->sk); list_add(&subflow->node, &msk->conn_list); sock_hold(ssock->sk); subflow->request_mptcp = 1; subflow->subflow_id = msk->subflow_id++; /* This is the first subflow, always with id 0 */ WRITE_ONCE(subflow->local_id, 0); mptcp_sock_graft(msk->first, sk->sk_socket); iput(SOCK_INODE(ssock)); return 0; } /* If the MPC handshake is not started, returns the first subflow, * eventually allocating it. */ struct sock *__mptcp_nmpc_sk(struct mptcp_sock *msk) { struct sock *sk = (struct sock *)msk; int ret; if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) return ERR_PTR(-EINVAL); if (!msk->first) { ret = __mptcp_socket_create(msk); if (ret) return ERR_PTR(ret); } return msk->first; } static void mptcp_drop(struct sock *sk, struct sk_buff *skb) { sk_drops_skbadd(sk, skb); __kfree_skb(skb); } static bool __mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, struct sk_buff *from, bool *fragstolen, int *delta) { int limit = READ_ONCE(sk->sk_rcvbuf); if (unlikely(MPTCP_SKB_CB(to)->cant_coalesce) || MPTCP_SKB_CB(from)->offset || ((to->len + from->len) > (limit >> 3)) || !skb_try_coalesce(to, from, fragstolen, delta)) return false; pr_debug("colesced seq %llx into %llx new len %d new end seq %llx\n", MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq, to->len, MPTCP_SKB_CB(from)->end_seq); MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq; return true; } static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, struct sk_buff *from) { bool fragstolen; int delta; if (!__mptcp_try_coalesce(sk, to, from, &fragstolen, &delta)) return false; /* note the fwd memory can reach a negative value after accounting * for the delta, but the later skb free will restore a non * negative one */ atomic_add(delta, &sk->sk_rmem_alloc); sk_mem_charge(sk, delta); kfree_skb_partial(from, fragstolen); return true; } static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, struct sk_buff *from) { if (MPTCP_SKB_CB(from)->map_seq != MPTCP_SKB_CB(to)->end_seq) return false; return mptcp_try_coalesce((struct sock *)msk, to, from); } /* "inspired" by tcp_rcvbuf_grow(), main difference: * - mptcp does not maintain a msk-level window clamp * - returns true when the receive buffer is actually updated */ static bool mptcp_rcvbuf_grow(struct sock *sk, u32 newval) { struct mptcp_sock *msk = mptcp_sk(sk); const struct net *net = sock_net(sk); u32 rcvwin, rcvbuf, cap, oldval; u64 grow; oldval = msk->rcvq_space.space; msk->rcvq_space.space = newval; if (!READ_ONCE(net->ipv4.sysctl_tcp_moderate_rcvbuf) || (sk->sk_userlocks & SOCK_RCVBUF_LOCK)) return false; /* DRS is always one RTT late. */ rcvwin = newval << 1; /* slow start: allow the sender to double its rate. */ grow = (u64)rcvwin * (newval - oldval); do_div(grow, oldval); rcvwin += grow << 1; if (!RB_EMPTY_ROOT(&msk->out_of_order_queue)) rcvwin += MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq - msk->ack_seq; cap = READ_ONCE(net->ipv4.sysctl_tcp_rmem[2]); rcvbuf = min_t(u32, mptcp_space_from_win(sk, rcvwin), cap); if (rcvbuf > sk->sk_rcvbuf) { WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); return true; } return false; } /* "inspired" by tcp_data_queue_ofo(), main differences: * - use mptcp seqs * - don't cope with sacks */ static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) { struct sock *sk = (struct sock *)msk; struct rb_node **p, *parent; u64 seq, end_seq, max_seq; struct sk_buff *skb1; seq = MPTCP_SKB_CB(skb)->map_seq; end_seq = MPTCP_SKB_CB(skb)->end_seq; max_seq = atomic64_read(&msk->rcv_wnd_sent); pr_debug("msk=%p seq=%llx limit=%llx empty=%d\n", msk, seq, max_seq, RB_EMPTY_ROOT(&msk->out_of_order_queue)); if (after64(end_seq, max_seq)) { /* out of window */ mptcp_drop(sk, skb); pr_debug("oow by %lld, rcv_wnd_sent %llu\n", (unsigned long long)end_seq - (unsigned long)max_seq, (unsigned long long)atomic64_read(&msk->rcv_wnd_sent)); MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW); return; } p = &msk->out_of_order_queue.rb_node; MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE); if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { rb_link_node(&skb->rbnode, NULL, p); rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); msk->ooo_last_skb = skb; goto end; } /* with 2 subflows, adding at end of ooo queue is quite likely * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup. */ if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); return; } /* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */ if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); parent = &msk->ooo_last_skb->rbnode; p = &parent->rb_right; goto insert; } /* Find place to insert this segment. Handle overlaps on the way. */ parent = NULL; while (*p) { parent = *p; skb1 = rb_to_skb(parent); if (before64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { p = &parent->rb_left; continue; } if (before64(seq, MPTCP_SKB_CB(skb1)->end_seq)) { if (!after64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) { /* All the bits are present. Drop. */ mptcp_drop(sk, skb); MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); return; } if (after64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { /* partial overlap: * | skb | * | skb1 | * continue traversing */ } else { /* skb's seq == skb1's seq and skb covers skb1. * Replace skb1 with skb. */ rb_replace_node(&skb1->rbnode, &skb->rbnode, &msk->out_of_order_queue); mptcp_drop(sk, skb1); MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); goto merge_right; } } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); return; } p = &parent->rb_right; } insert: /* Insert segment into RB tree. */ rb_link_node(&skb->rbnode, parent, p); rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); merge_right: /* Remove other segments covered by skb. */ while ((skb1 = skb_rb_next(skb)) != NULL) { if (before64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) break; rb_erase(&skb1->rbnode, &msk->out_of_order_queue); mptcp_drop(sk, skb1); MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); } /* If there is no skb after us, we are the last_skb ! */ if (!skb1) msk->ooo_last_skb = skb; end: skb_condense(skb); skb_set_owner_r(skb, sk); /* do not grow rcvbuf for not-yet-accepted or orphaned sockets. */ if (sk->sk_socket) mptcp_rcvbuf_grow(sk, msk->rcvq_space.space); } static void mptcp_init_skb(struct sock *ssk, struct sk_buff *skb, int offset, int copy_len) { const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); bool has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp; /* the skb map_seq accounts for the skb offset: * mptcp_subflow_get_mapped_dsn() is based on the current tp->copied_seq * value */ MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow); MPTCP_SKB_CB(skb)->end_seq = MPTCP_SKB_CB(skb)->map_seq + copy_len; MPTCP_SKB_CB(skb)->offset = offset; MPTCP_SKB_CB(skb)->has_rxtstamp = has_rxtstamp; MPTCP_SKB_CB(skb)->cant_coalesce = 0; __skb_unlink(skb, &ssk->sk_receive_queue); skb_ext_reset(skb); skb_dst_drop(skb); } static bool __mptcp_move_skb(struct sock *sk, struct sk_buff *skb) { u64 copy_len = MPTCP_SKB_CB(skb)->end_seq - MPTCP_SKB_CB(skb)->map_seq; struct mptcp_sock *msk = mptcp_sk(sk); struct sk_buff *tail; /* try to fetch required memory from subflow */ if (!sk_rmem_schedule(sk, skb, skb->truesize)) { MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RCVPRUNED); goto drop; } if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { /* in sequence */ msk->bytes_received += copy_len; WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); tail = skb_peek_tail(&sk->sk_receive_queue); if (tail && mptcp_try_coalesce(sk, tail, skb)) return true; skb_set_owner_r(skb, sk); __skb_queue_tail(&sk->sk_receive_queue, skb); return true; } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { mptcp_data_queue_ofo(msk, skb); return false; } /* old data, keep it simple and drop the whole pkt, sender * will retransmit as needed, if needed. */ MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); drop: mptcp_drop(sk, skb); return false; } static void mptcp_stop_rtx_timer(struct sock *sk) { struct inet_connection_sock *icsk = inet_csk(sk); sk_stop_timer(sk, &icsk->icsk_retransmit_timer); mptcp_sk(sk)->timer_ival = 0; } static void mptcp_close_wake_up(struct sock *sk) { if (sock_flag(sk, SOCK_DEAD)) return; sk->sk_state_change(sk); if (sk->sk_shutdown == SHUTDOWN_MASK || sk->sk_state == TCP_CLOSE) sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); else sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); } static void mptcp_shutdown_subflows(struct mptcp_sock *msk) { struct mptcp_subflow_context *subflow; mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); bool slow; slow = lock_sock_fast(ssk); tcp_shutdown(ssk, SEND_SHUTDOWN); unlock_sock_fast(ssk, slow); } } /* called under the msk socket lock */ static bool mptcp_pending_data_fin_ack(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); return ((1 << sk->sk_state) & (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) && msk->write_seq == READ_ONCE(msk->snd_una); } static void mptcp_check_data_fin_ack(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); /* Look for an acknowledged DATA_FIN */ if (mptcp_pending_data_fin_ack(sk)) { WRITE_ONCE(msk->snd_data_fin_enable, 0); switch (sk->sk_state) { case TCP_FIN_WAIT1: mptcp_set_state(sk, TCP_FIN_WAIT2); break; case TCP_CLOSING: case TCP_LAST_ACK: mptcp_shutdown_subflows(msk); mptcp_set_state(sk, TCP_CLOSE); break; } mptcp_close_wake_up(sk); } } /* can be called with no lock acquired */ static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) { struct mptcp_sock *msk = mptcp_sk(sk); if (READ_ONCE(msk->rcv_data_fin) && ((1 << inet_sk_state_load(sk)) & (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) { u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); if (READ_ONCE(msk->ack_seq) == rcv_data_fin_seq) { if (seq) *seq = rcv_data_fin_seq; return true; } } return false; } static void mptcp_set_datafin_timeout(struct sock *sk) { struct inet_connection_sock *icsk = inet_csk(sk); u32 retransmits; retransmits = min_t(u32, icsk->icsk_retransmits, ilog2(TCP_RTO_MAX / TCP_RTO_MIN)); mptcp_sk(sk)->timer_ival = TCP_RTO_MIN << retransmits; } static void __mptcp_set_timeout(struct sock *sk, long tout) { mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; } static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subflow) { const struct sock *ssk = mptcp_subflow_tcp_sock(subflow); return inet_csk(ssk)->icsk_pending && !subflow->stale_count ? icsk_timeout(inet_csk(ssk)) - jiffies : 0; } static void mptcp_set_timeout(struct sock *sk) { struct mptcp_subflow_context *subflow; long tout = 0; mptcp_for_each_subflow(mptcp_sk(sk), subflow) tout = max(tout, mptcp_timeout_from_subflow(subflow)); __mptcp_set_timeout(sk, tout); } static inline bool tcp_can_send_ack(const struct sock *ssk) { return !((1 << inet_sk_state_load(ssk)) & (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE | TCPF_LISTEN)); } void __mptcp_subflow_send_ack(struct sock *ssk) { if (tcp_can_send_ack(ssk)) tcp_send_ack(ssk); } static void mptcp_subflow_send_ack(struct sock *ssk) { bool slow; slow = lock_sock_fast(ssk); __mptcp_subflow_send_ack(ssk); unlock_sock_fast(ssk, slow); } static void mptcp_send_ack(struct mptcp_sock *msk) { struct mptcp_subflow_context *subflow; mptcp_for_each_subflow(msk, subflow) mptcp_subflow_send_ack(mptcp_subflow_tcp_sock(subflow)); } static void mptcp_subflow_cleanup_rbuf(struct sock *ssk, int copied) { bool slow; slow = lock_sock_fast(ssk); if (tcp_can_send_ack(ssk)) tcp_cleanup_rbuf(ssk, copied); unlock_sock_fast(ssk, slow); } static bool mptcp_subflow_could_cleanup(const struct sock *ssk, bool rx_empty) { const struct inet_connection_sock *icsk = inet_csk(ssk); u8 ack_pending = READ_ONCE(icsk->icsk_ack.pending); const struct tcp_sock *tp = tcp_sk(ssk); return (ack_pending & ICSK_ACK_SCHED) && ((READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->rcv_wup) > READ_ONCE(icsk->icsk_ack.rcv_mss)) || (rx_empty && ack_pending & (ICSK_ACK_PUSHED2 | ICSK_ACK_PUSHED))); } static void mptcp_cleanup_rbuf(struct mptcp_sock *msk, int copied) { int old_space = READ_ONCE(msk->old_wspace); struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; int space = __mptcp_space(sk); bool cleanup, rx_empty; cleanup = (space > 0) && (space >= (old_space << 1)) && copied; rx_empty = !sk_rmem_alloc_get(sk) && copied; mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); if (cleanup || mptcp_subflow_could_cleanup(ssk, rx_empty)) mptcp_subflow_cleanup_rbuf(ssk, copied); } } static void mptcp_check_data_fin(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); u64 rcv_data_fin_seq; /* Need to ack a DATA_FIN received from a peer while this side * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2. * msk->rcv_data_fin was set when parsing the incoming options * at the subflow level and the msk lock was not held, so this * is the first opportunity to act on the DATA_FIN and change * the msk state. * * If we are caught up to the sequence number of the incoming * DATA_FIN, send the DATA_ACK now and do state transition. If * not caught up, do nothing and let the recv code send DATA_ACK * when catching up. */ if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); WRITE_ONCE(msk->rcv_data_fin, 0); WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | RCV_SHUTDOWN); smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ switch (sk->sk_state) { case TCP_ESTABLISHED: mptcp_set_state(sk, TCP_CLOSE_WAIT); break; case TCP_FIN_WAIT1: mptcp_set_state(sk, TCP_CLOSING); break; case TCP_FIN_WAIT2: mptcp_shutdown_subflows(msk); mptcp_set_state(sk, TCP_CLOSE); break; default: /* Other states not expected */ WARN_ON_ONCE(1); break; } if (!__mptcp_check_fallback(msk)) mptcp_send_ack(msk); mptcp_close_wake_up(sk); } } static void mptcp_dss_corruption(struct mptcp_sock *msk, struct sock *ssk) { if (!mptcp_try_fallback(ssk, MPTCP_MIB_DSSCORRUPTIONFALLBACK)) { MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_DSSCORRUPTIONRESET); mptcp_subflow_reset(ssk); } } static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, struct sock *ssk) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); struct sock *sk = (struct sock *)msk; bool more_data_avail; struct tcp_sock *tp; bool ret = false; pr_debug("msk=%p ssk=%p\n", msk, ssk); tp = tcp_sk(ssk); do { u32 map_remaining, offset; u32 seq = tp->copied_seq; struct sk_buff *skb; bool fin; if (sk_rmem_alloc_get(sk) > sk->sk_rcvbuf) break; /* try to move as much data as available */ map_remaining = subflow->map_data_len - mptcp_subflow_get_map_offset(subflow); skb = skb_peek(&ssk->sk_receive_queue); if (unlikely(!skb)) break; if (__mptcp_check_fallback(msk)) { /* Under fallback skbs have no MPTCP extension and TCP could * collapse them between the dummy map creation and the * current dequeue. Be sure to adjust the map size. */ map_remaining = skb->len; subflow->map_data_len = skb->len; } offset = seq - TCP_SKB_CB(skb)->seq; fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; if (fin) seq++; if (offset < skb->len) { size_t len = skb->len - offset; mptcp_init_skb(ssk, skb, offset, len); skb_orphan(skb); ret = __mptcp_move_skb(sk, skb) || ret; seq += len; if (unlikely(map_remaining < len)) { DEBUG_NET_WARN_ON_ONCE(1); mptcp_dss_corruption(msk, ssk); } } else { if (unlikely(!fin)) { DEBUG_NET_WARN_ON_ONCE(1); mptcp_dss_corruption(msk, ssk); } sk_eat_skb(ssk, skb); } WRITE_ONCE(tp->copied_seq, seq); more_data_avail = mptcp_subflow_data_available(ssk); } while (more_data_avail); if (ret) msk->last_data_recv = tcp_jiffies32; return ret; } static bool __mptcp_ofo_queue(struct mptcp_sock *msk) { struct sock *sk = (struct sock *)msk; struct sk_buff *skb, *tail; bool moved = false; struct rb_node *p; u64 end_seq; p = rb_first(&msk->out_of_order_queue); pr_debug("msk=%p empty=%d\n", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); while (p) { skb = rb_to_skb(p); if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) break; p = rb_next(p); rb_erase(&skb->rbnode, &msk->out_of_order_queue); if (unlikely(!after64(MPTCP_SKB_CB(skb)->end_seq, msk->ack_seq))) { mptcp_drop(sk, skb); MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); continue; } end_seq = MPTCP_SKB_CB(skb)->end_seq; tail = skb_peek_tail(&sk->sk_receive_queue); if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; /* skip overlapping data, if any */ pr_debug("uncoalesced seq=%llx ack seq=%llx delta=%d\n", MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, delta); MPTCP_SKB_CB(skb)->offset += delta; MPTCP_SKB_CB(skb)->map_seq += delta; __skb_queue_tail(&sk->sk_receive_queue, skb); } msk->bytes_received += end_seq - msk->ack_seq; WRITE_ONCE(msk->ack_seq, end_seq); moved = true; } return moved; } static bool __mptcp_subflow_error_report(struct sock *sk, struct sock *ssk) { int err = sock_error(ssk); int ssk_state; if (!err) return false; /* only propagate errors on fallen-back sockets or * on MPC connect */ if (sk->sk_state != TCP_SYN_SENT && !__mptcp_check_fallback(mptcp_sk(sk))) return false; /* We need to propagate only transition to CLOSE state. * Orphaned socket will see such state change via * subflow_sched_work_if_closed() and that path will properly * destroy the msk as needed. */ ssk_state = inet_sk_state_load(ssk); if (ssk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DEAD)) mptcp_set_state(sk, ssk_state); WRITE_ONCE(sk->sk_err, -err); /* This barrier is coupled with smp_rmb() in mptcp_poll() */ smp_wmb(); sk_error_report(sk); return true; } void __mptcp_error_report(struct sock *sk) { struct mptcp_subflow_context *subflow; struct mptcp_sock *msk = mptcp_sk(sk); mptcp_for_each_subflow(msk, subflow) if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow))) break; } /* In most cases we will be able to lock the mptcp socket. If its already * owned, we need to defer to the work queue to avoid ABBA deadlock. */ static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) { struct sock *sk = (struct sock *)msk; bool moved; moved = __mptcp_move_skbs_from_subflow(msk, ssk); __mptcp_ofo_queue(msk); if (unlikely(ssk->sk_err)) __mptcp_subflow_error_report(sk, ssk); /* If the moves have caught up with the DATA_FIN sequence number * it's time to ack the DATA_FIN and change socket state, but * this is not a good place to change state. Let the workqueue * do it. */ if (mptcp_pending_data_fin(sk, NULL)) mptcp_schedule_work(sk); return moved; } static void __mptcp_data_ready(struct sock *sk, struct sock *ssk) { struct mptcp_sock *msk = mptcp_sk(sk); /* Wake-up the reader only for in-sequence data */ if (move_skbs_to_msk(msk, ssk) && mptcp_epollin_ready(sk)) sk->sk_data_ready(sk); } void mptcp_data_ready(struct sock *sk, struct sock *ssk) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); /* The peer can send data while we are shutting down this * subflow at msk destruction time, but we must avoid enqueuing * more data to the msk receive queue */ if (unlikely(subflow->disposable)) return; mptcp_data_lock(sk); if (!sock_owned_by_user(sk)) __mptcp_data_ready(sk, ssk); else __set_bit(MPTCP_DEQUEUE, &mptcp_sk(sk)->cb_flags); mptcp_data_unlock(sk); } static void mptcp_subflow_joined(struct mptcp_sock *msk, struct sock *ssk) { mptcp_subflow_ctx(ssk)->map_seq = READ_ONCE(msk->ack_seq); msk->allow_infinite_fallback = false; mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); } static bool __mptcp_finish_join(struct mptcp_sock *msk, struct sock *ssk) { struct sock *sk = (struct sock *)msk; if (sk->sk_state != TCP_ESTABLISHED) return false; spin_lock_bh(&msk->fallback_lock); if (!msk->allow_subflows) { spin_unlock_bh(&msk->fallback_lock); return false; } mptcp_subflow_joined(msk, ssk); spin_unlock_bh(&msk->fallback_lock); /* attach to msk socket only after we are sure we will deal with it * at close time */ if (sk->sk_socket && !ssk->sk_socket) mptcp_sock_graft(ssk, sk->sk_socket); mptcp_subflow_ctx(ssk)->subflow_id = msk->subflow_id++; mptcp_sockopt_sync_locked(msk, ssk); mptcp_stop_tout_timer(sk); __mptcp_propagate_sndbuf(sk, ssk); return true; } static void __mptcp_flush_join_list(struct sock *sk, struct list_head *join_list) { struct mptcp_subflow_context *tmp, *subflow; struct mptcp_sock *msk = mptcp_sk(sk); list_for_each_entry_safe(subflow, tmp, join_list, node) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); bool slow = lock_sock_fast(ssk); list_move_tail(&subflow->node, &msk->conn_list); if (!__mptcp_finish_join(msk, ssk)) mptcp_subflow_reset(ssk); unlock_sock_fast(ssk, slow); } } static bool mptcp_rtx_timer_pending(struct sock *sk) { return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); } static void mptcp_reset_rtx_timer(struct sock *sk) { struct inet_connection_sock *icsk = inet_csk(sk); unsigned long tout; /* prevent rescheduling on close */ if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) return; tout = mptcp_sk(sk)->timer_ival; sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); } bool mptcp_schedule_work(struct sock *sk) { if (inet_sk_state_load(sk) == TCP_CLOSE) return false; /* Get a reference on this socket, mptcp_worker() will release it. * As mptcp_worker() might complete before us, we can not avoid * a sock_hold()/sock_put() if schedule_work() returns false. */ sock_hold(sk); if (schedule_work(&mptcp_sk(sk)->work)) return true; sock_put(sk); return false; } static bool mptcp_skb_can_collapse_to(u64 write_seq, const struct sk_buff *skb, const struct mptcp_ext *mpext) { if (!tcp_skb_can_collapse_to(skb)) return false; /* can collapse only if MPTCP level sequence is in order and this * mapping has not been xmitted yet */ return mpext && mpext->data_seq + mpext->data_len == write_seq && !mpext->frozen; } /* we can append data to the given data frag if: * - there is space available in the backing page_frag * - the data frag tail matches the current page_frag free offset * - the data frag end sequence number matches the current write seq */ static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, const struct page_frag *pfrag, const struct mptcp_data_frag *df) { return df && pfrag->page == df->page && pfrag->size - pfrag->offset > 0 && pfrag->offset == (df->offset + df->data_len) && df->data_seq + df->data_len == msk->write_seq; } static void dfrag_uncharge(struct sock *sk, int len) { sk_mem_uncharge(sk, len); sk_wmem_queued_add(sk, -len); } static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) { int len = dfrag->data_len + dfrag->overhead; list_del(&dfrag->list); dfrag_uncharge(sk, len); put_page(dfrag->page); } /* called under both the msk socket lock and the data lock */ static void __mptcp_clean_una(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); struct mptcp_data_frag *dtmp, *dfrag; u64 snd_una; snd_una = msk->snd_una; list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { if (after64(dfrag->data_seq + dfrag->data_len, snd_una)) break; if (unlikely(dfrag == msk->first_pending)) { /* in recovery mode can see ack after the current snd head */ if (WARN_ON_ONCE(!msk->recovery)) break; msk->first_pending = mptcp_send_next(sk); } dfrag_clear(sk, dfrag); } dfrag = mptcp_rtx_head(sk); if (dfrag && after64(snd_una, dfrag->data_seq)) { u64 delta = snd_una - dfrag->data_seq; /* prevent wrap around in recovery mode */ if (unlikely(delta > dfrag->already_sent)) { if (WARN_ON_ONCE(!msk->recovery)) goto out; if (WARN_ON_ONCE(delta > dfrag->data_len)) goto out; dfrag->already_sent += delta - dfrag->already_sent; } dfrag->data_seq += delta; dfrag->offset += delta; dfrag->data_len -= delta; dfrag->already_sent -= delta; dfrag_uncharge(sk, delta); } /* all retransmitted data acked, recovery completed */ if (unlikely(msk->recovery) && after64(msk->snd_una, msk->recovery_snd_nxt)) msk->recovery = false; out: if (snd_una == msk->snd_nxt && snd_una == msk->write_seq) { if (mptcp_rtx_timer_pending(sk) && !mptcp_data_fin_enabled(msk)) mptcp_stop_rtx_timer(sk); } else { mptcp_reset_rtx_timer(sk); } if (mptcp_pending_data_fin_ack(sk)) mptcp_schedule_work(sk); } static void __mptcp_clean_una_wakeup(struct sock *sk) { lockdep_assert_held_once(&sk->sk_lock.slock); __mptcp_clean_una(sk); mptcp_write_space(sk); } static void mptcp_clean_una_wakeup(struct sock *sk) { mptcp_data_lock(sk); __mptcp_clean_una_wakeup(sk); mptcp_data_unlock(sk); } static void mptcp_enter_memory_pressure(struct sock *sk) { struct mptcp_subflow_context *subflow; struct mptcp_sock *msk = mptcp_sk(sk); bool first = true; mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); if (first) tcp_enter_memory_pressure(ssk); sk_stream_moderate_sndbuf(ssk); first = false; } __mptcp_sync_sndbuf(sk); } /* ensure we get enough memory for the frag hdr, beyond some minimal amount of * data */ static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) { if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag), pfrag, sk->sk_allocation))) return true; mptcp_enter_memory_pressure(sk); return false; } static struct mptcp_data_frag * mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, int orig_offset) { int offset = ALIGN(orig_offset, sizeof(long)); struct mptcp_data_frag *dfrag; dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset); dfrag->data_len = 0; dfrag->data_seq = msk->write_seq; dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag); dfrag->offset = offset + sizeof(struct mptcp_data_frag); dfrag->already_sent = 0; dfrag->page = pfrag->page; return dfrag; } struct mptcp_sendmsg_info { int mss_now; int size_goal; u16 limit; u16 sent; unsigned int flags; bool data_lock_held; }; static int mptcp_check_allowed_size(const struct mptcp_sock *msk, struct sock *ssk, u64 data_seq, int avail_size) { u64 window_end = mptcp_wnd_end(msk); u64 mptcp_snd_wnd; if (__mptcp_check_fallback(msk)) return avail_size; mptcp_snd_wnd = window_end - data_seq; avail_size = min_t(unsigned int, mptcp_snd_wnd, avail_size); if (unlikely(tcp_sk(ssk)->snd_wnd < mptcp_snd_wnd)) { tcp_sk(ssk)->snd_wnd = min_t(u64, U32_MAX, mptcp_snd_wnd); MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_SNDWNDSHARED); } return avail_size; } static bool __mptcp_add_ext(struct sk_buff *skb, gfp_t gfp) { struct skb_ext *mpext = __skb_ext_alloc(gfp); if (!mpext) return false; __skb_ext_set(skb, SKB_EXT_MPTCP, mpext); return true; } static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp) { struct sk_buff *skb; skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp); if (likely(skb)) { if (likely(__mptcp_add_ext(skb, gfp))) { skb_reserve(skb, MAX_TCP_HEADER); skb->ip_summed = CHECKSUM_PARTIAL; INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); return skb; } __kfree_skb(skb); } else { mptcp_enter_memory_pressure(sk); } return NULL; } static struct sk_buff *__mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp) { struct sk_buff *skb; skb = __mptcp_do_alloc_tx_skb(sk, gfp); if (!skb) return NULL; if (likely(sk_wmem_schedule(ssk, skb->truesize))) { tcp_skb_entail(ssk, skb); return skb; } tcp_skb_tsorted_anchor_cleanup(skb); kfree_skb(skb); return NULL; } static struct sk_buff *mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, bool data_lock_held) { gfp_t gfp = data_lock_held ? GFP_ATOMIC : sk->sk_allocation; return __mptcp_alloc_tx_skb(sk, ssk, gfp); } /* note: this always recompute the csum on the whole skb, even * if we just appended a single frag. More status info needed */ static void mptcp_update_data_checksum(struct sk_buff *skb, int added) { struct mptcp_ext *mpext = mptcp_get_ext(skb); __wsum csum = ~csum_unfold(mpext->csum); int offset = skb->len - added; mpext->csum = csum_fold(csum_block_add(csum, skb_checksum(skb, offset, added, 0), offset)); } static void mptcp_update_infinite_map(struct mptcp_sock *msk, struct sock *ssk, struct mptcp_ext *mpext) { if (!mpext) return; mpext->infinite_map = 1; mpext->data_len = 0; if (!mptcp_try_fallback(ssk, MPTCP_MIB_INFINITEMAPTX)) { MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_FALLBACKFAILED); mptcp_subflow_reset(ssk); return; } mptcp_subflow_ctx(ssk)->send_infinite_map = 0; } #define MPTCP_MAX_GSO_SIZE (GSO_LEGACY_MAX_SIZE - (MAX_TCP_HEADER + 1)) static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, struct mptcp_data_frag *dfrag, struct mptcp_sendmsg_info *info) { u64 data_seq = dfrag->data_seq + info->sent; int offset = dfrag->offset + info->sent; struct mptcp_sock *msk = mptcp_sk(sk); bool zero_window_probe = false; struct mptcp_ext *mpext = NULL; bool can_coalesce = false; bool reuse_skb = true; struct sk_buff *skb; size_t copy; int i; pr_debug("msk=%p ssk=%p sending dfrag at seq=%llu len=%u already sent=%u\n", msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); if (WARN_ON_ONCE(info->sent > info->limit || info->limit > dfrag->data_len)) return 0; if (unlikely(!__tcp_can_send(ssk))) return -EAGAIN; /* compute send limit */ if (unlikely(ssk->sk_gso_max_size > MPTCP_MAX_GSO_SIZE)) ssk->sk_gso_max_size = MPTCP_MAX_GSO_SIZE; info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags); copy = info->size_goal; skb = tcp_write_queue_tail(ssk); if (skb && copy > skb->len) { /* Limit the write to the size available in the * current skb, if any, so that we create at most a new skb. * Explicitly tells TCP internals to avoid collapsing on later * queue management operation, to avoid breaking the ext <-> * SSN association set here */ mpext = mptcp_get_ext(skb); if (!mptcp_skb_can_collapse_to(data_seq, skb, mpext)) { TCP_SKB_CB(skb)->eor = 1; tcp_mark_push(tcp_sk(ssk), skb); goto alloc_skb; } i = skb_shinfo(skb)->nr_frags; can_coalesce = skb_can_coalesce(skb, i, dfrag->page, offset); if (!can_coalesce && i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) { tcp_mark_push(tcp_sk(ssk), skb); goto alloc_skb; } copy -= skb->len; } else { alloc_skb: skb = mptcp_alloc_tx_skb(sk, ssk, info->data_lock_held); if (!skb) return -ENOMEM; i = skb_shinfo(skb)->nr_frags; reuse_skb = false; mpext = mptcp_get_ext(skb); } /* Zero window and all data acked? Probe. */ copy = mptcp_check_allowed_size(msk, ssk, data_seq, copy); if (copy == 0) { u64 snd_una = READ_ONCE(msk->snd_una); /* No need for zero probe if there are any data pending * either at the msk or ssk level; skb is the current write * queue tail and can be empty at this point. */ if (snd_una != msk->snd_nxt || skb->len || skb != tcp_send_head(ssk)) { tcp_remove_empty_skb(ssk); return 0; } zero_window_probe = true; data_seq = snd_una - 1; copy = 1; } copy = min_t(size_t, copy, info->limit - info->sent); if (!sk_wmem_schedule(ssk, copy)) { tcp_remove_empty_skb(ssk); return -ENOMEM; } if (can_coalesce) { skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); } else { get_page(dfrag->page); skb_fill_page_desc(skb, i, dfrag->page, offset, copy); } skb->len += copy; skb->data_len += copy; skb->truesize += copy; sk_wmem_queued_add(ssk, copy); sk_mem_charge(ssk, copy); WRITE_ONCE(tcp_sk(ssk)->write_seq, tcp_sk(ssk)->write_seq + copy); TCP_SKB_CB(skb)->end_seq += copy; tcp_skb_pcount_set(skb, 0); /* on skb reuse we just need to update the DSS len */ if (reuse_skb) { TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; mpext->data_len += copy; goto out; } memset(mpext, 0, sizeof(*mpext)); mpext->data_seq = data_seq; mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; mpext->data_len = copy; mpext->use_map = 1; mpext->dsn64 = 1; pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d\n", mpext->data_seq, mpext->subflow_seq, mpext->data_len, mpext->dsn64); if (zero_window_probe) { MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_WINPROBE); mptcp_subflow_ctx(ssk)->rel_write_seq += copy; mpext->frozen = 1; if (READ_ONCE(msk->csum_enabled)) mptcp_update_data_checksum(skb, copy); tcp_push_pending_frames(ssk); return 0; } out: if (READ_ONCE(msk->csum_enabled)) mptcp_update_data_checksum(skb, copy); if (mptcp_subflow_ctx(ssk)->send_infinite_map) mptcp_update_infinite_map(msk, ssk, mpext); trace_mptcp_sendmsg_frag(mpext); mptcp_subflow_ctx(ssk)->rel_write_seq += copy; return copy; } #define MPTCP_SEND_BURST_SIZE ((1 << 16) - \ sizeof(struct tcphdr) - \ MAX_TCP_OPTION_SPACE - \ sizeof(struct ipv6hdr) - \ sizeof(struct frag_hdr)) struct subflow_send_info { struct sock *ssk; u64 linger_time; }; void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow) { if (!subflow->stale) return; subflow->stale = 0; MPTCP_INC_STATS(sock_net(mptcp_subflow_tcp_sock(subflow)), MPTCP_MIB_SUBFLOWRECOVER); } bool mptcp_subflow_active(struct mptcp_subflow_context *subflow) { if (unlikely(subflow->stale)) { u32 rcv_tstamp = READ_ONCE(tcp_sk(mptcp_subflow_tcp_sock(subflow))->rcv_tstamp); if (subflow->stale_rcv_tstamp == rcv_tstamp) return false; mptcp_subflow_set_active(subflow); } return __mptcp_subflow_active(subflow); } #define SSK_MODE_ACTIVE 0 #define SSK_MODE_BACKUP 1 #define SSK_MODE_MAX 2 /* implement the mptcp packet scheduler; * returns the subflow that will transmit the next DSS * additionally updates the rtx timeout */ struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) { struct subflow_send_info send_info[SSK_MODE_MAX]; struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; u32 pace, burst, wmem; int i, nr_active = 0; struct sock *ssk; u64 linger_time; long tout = 0; /* pick the subflow with the lower wmem/wspace ratio */ for (i = 0; i < SSK_MODE_MAX; ++i) { send_info[i].ssk = NULL; send_info[i].linger_time = -1; } mptcp_for_each_subflow(msk, subflow) { bool backup = subflow->backup || subflow->request_bkup; trace_mptcp_subflow_get_send(subflow); ssk = mptcp_subflow_tcp_sock(subflow); if (!mptcp_subflow_active(subflow)) continue; tout = max(tout, mptcp_timeout_from_subflow(subflow)); nr_active += !backup; pace = subflow->avg_pacing_rate; if (unlikely(!pace)) { /* init pacing rate from socket */ subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate); pace = subflow->avg_pacing_rate; if (!pace) continue; } linger_time = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, pace); if (linger_time < send_info[backup].linger_time) { send_info[backup].ssk = ssk; send_info[backup].linger_time = linger_time; } } __mptcp_set_timeout(sk, tout); /* pick the best backup if no other subflow is active */ if (!nr_active) send_info[SSK_MODE_ACTIVE].ssk = send_info[SSK_MODE_BACKUP].ssk; /* According to the blest algorithm, to avoid HoL blocking for the * faster flow, we need to: * - estimate the faster flow linger time * - use the above to estimate the amount of byte transferred * by the faster flow * - check that the amount of queued data is greater than the above, * otherwise do not use the picked, slower, subflow * We select the subflow with the shorter estimated time to flush * the queued mem, which basically ensure the above. We just need * to check that subflow has a non empty cwin. */ ssk = send_info[SSK_MODE_ACTIVE].ssk; if (!ssk || !sk_stream_memory_free(ssk)) return NULL; burst = min_t(int, MPTCP_SEND_BURST_SIZE, mptcp_wnd_end(msk) - msk->snd_nxt); wmem = READ_ONCE(ssk->sk_wmem_queued); if (!burst) return ssk; subflow = mptcp_subflow_ctx(ssk); subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem + READ_ONCE(ssk->sk_pacing_rate) * burst, burst + wmem); msk->snd_burst = burst; return ssk; } static void mptcp_push_release(struct sock *ssk, struct mptcp_sendmsg_info *info) { tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal); release_sock(ssk); } static void mptcp_update_post_push(struct mptcp_sock *msk, struct mptcp_data_frag *dfrag, u32 sent) { u64 snd_nxt_new = dfrag->data_seq; dfrag->already_sent += sent; msk->snd_burst -= sent; snd_nxt_new += dfrag->already_sent; /* snd_nxt_new can be smaller than snd_nxt in case mptcp * is recovering after a failover. In that event, this re-sends * old segments. * * Thus compute snd_nxt_new candidate based on * the dfrag->data_seq that was sent and the data * that has been handed to the subflow for transmission * and skip update in case it was old dfrag. */ if (likely(after64(snd_nxt_new, msk->snd_nxt))) { msk->bytes_sent += snd_nxt_new - msk->snd_nxt; WRITE_ONCE(msk->snd_nxt, snd_nxt_new); } } void mptcp_check_and_set_pending(struct sock *sk) { if (mptcp_send_head(sk)) { mptcp_data_lock(sk); mptcp_sk(sk)->cb_flags |= BIT(MPTCP_PUSH_PENDING); mptcp_data_unlock(sk); } } static int __subflow_push_pending(struct sock *sk, struct sock *ssk, struct mptcp_sendmsg_info *info) { struct mptcp_sock *msk = mptcp_sk(sk); struct mptcp_data_frag *dfrag; int len, copied = 0, err = 0; while ((dfrag = mptcp_send_head(sk))) { info->sent = dfrag->already_sent; info->limit = dfrag->data_len; len = dfrag->data_len - dfrag->already_sent; while (len > 0) { int ret = 0; ret = mptcp_sendmsg_frag(sk, ssk, dfrag, info); if (ret <= 0) { err = copied ? : ret; goto out; } info->sent += ret; copied += ret; len -= ret; mptcp_update_post_push(msk, dfrag, ret); } msk->first_pending = mptcp_send_next(sk); if (msk->snd_burst <= 0 || !sk_stream_memory_free(ssk) || !mptcp_subflow_active(mptcp_subflow_ctx(ssk))) { err = copied; goto out; } mptcp_set_timeout(sk); } err = copied; out: if (err > 0) msk->last_data_sent = tcp_jiffies32; return err; } void __mptcp_push_pending(struct sock *sk, unsigned int flags) { struct sock *prev_ssk = NULL, *ssk = NULL; struct mptcp_sock *msk = mptcp_sk(sk); struct mptcp_sendmsg_info info = { .flags = flags, }; bool do_check_data_fin = false; int push_count = 1; while (mptcp_send_head(sk) && (push_count > 0)) { struct mptcp_subflow_context *subflow; int ret = 0; if (mptcp_sched_get_send(msk)) break; push_count = 0; mptcp_for_each_subflow(msk, subflow) { if (READ_ONCE(subflow->scheduled)) { mptcp_subflow_set_scheduled(subflow, false); prev_ssk = ssk; ssk = mptcp_subflow_tcp_sock(subflow); if (ssk != prev_ssk) { /* First check. If the ssk has changed since * the last round, release prev_ssk */ if (prev_ssk) mptcp_push_release(prev_ssk, &info); /* Need to lock the new subflow only if different * from the previous one, otherwise we are still * helding the relevant lock */ lock_sock(ssk); } push_count++; ret = __subflow_push_pending(sk, ssk, &info); if (ret <= 0) { if (ret != -EAGAIN || (1 << ssk->sk_state) & (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSE)) push_count--; continue; } do_check_data_fin = true; } } } /* at this point we held the socket lock for the last subflow we used */ if (ssk) mptcp_push_release(ssk, &info); /* ensure the rtx timer is running */ if (!mptcp_rtx_timer_pending(sk)) mptcp_reset_rtx_timer(sk); if (do_check_data_fin) mptcp_check_send_data_fin(sk); } static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool first) { struct mptcp_sock *msk = mptcp_sk(sk); struct mptcp_sendmsg_info info = { .data_lock_held = true, }; bool keep_pushing = true; struct sock *xmit_ssk; int copied = 0; info.flags = 0; while (mptcp_send_head(sk) && keep_pushing) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); int ret = 0; /* check for a different subflow usage only after * spooling the first chunk of data */ if (first) { mptcp_subflow_set_scheduled(subflow, false); ret = __subflow_push_pending(sk, ssk, &info); first = false; if (ret <= 0) break; copied += ret; continue; } if (mptcp_sched_get_send(msk)) goto out; if (READ_ONCE(subflow->scheduled)) { mptcp_subflow_set_scheduled(subflow, false); ret = __subflow_push_pending(sk, ssk, &info); if (ret <= 0) keep_pushing = false; copied += ret; } mptcp_for_each_subflow(msk, subflow) { if (READ_ONCE(subflow->scheduled)) { xmit_ssk = mptcp_subflow_tcp_sock(subflow); if (xmit_ssk != ssk) { mptcp_subflow_delegate(subflow, MPTCP_DELEGATE_SEND); keep_pushing = false; } } } } out: /* __mptcp_alloc_tx_skb could have released some wmem and we are * not going to flush it via release_sock() */ if (copied) { tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, info.size_goal); if (!mptcp_rtx_timer_pending(sk)) mptcp_reset_rtx_timer(sk); if (msk->snd_data_fin_enable && msk->snd_nxt + 1 == msk->write_seq) mptcp_schedule_work(sk); } } static int mptcp_disconnect(struct sock *sk, int flags); static int mptcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, size_t len, int *copied_syn) { unsigned int saved_flags = msg->msg_flags; struct mptcp_sock *msk = mptcp_sk(sk); struct sock *ssk; int ret; /* on flags based fastopen the mptcp is supposed to create the * first subflow right now. Otherwise we are in the defer_connect * path, and the first subflow must be already present. * Since the defer_connect flag is cleared after the first succsful * fastopen attempt, no need to check for additional subflow status. */ if (msg->msg_flags & MSG_FASTOPEN) { ssk = __mptcp_nmpc_sk(msk); if (IS_ERR(ssk)) return PTR_ERR(ssk); } if (!msk->first) return -EINVAL; ssk = msk->first; lock_sock(ssk); msg->msg_flags |= MSG_DONTWAIT; msk->fastopening = 1; ret = tcp_sendmsg_fastopen(ssk, msg, copied_syn, len, NULL); msk->fastopening = 0; msg->msg_flags = saved_flags; release_sock(ssk); /* do the blocking bits of inet_stream_connect outside the ssk socket lock */ if (ret == -EINPROGRESS && !(msg->msg_flags & MSG_DONTWAIT)) { ret = __inet_stream_connect(sk->sk_socket, msg->msg_name, msg->msg_namelen, msg->msg_flags, 1); /* Keep the same behaviour of plain TCP: zero the copied bytes in * case of any error, except timeout or signal */ if (ret && ret != -EINPROGRESS && ret != -ERESTARTSYS && ret != -EINTR) *copied_syn = 0; } else if (ret && ret != -EINPROGRESS) { /* The disconnect() op called by tcp_sendmsg_fastopen()/ * __inet_stream_connect() can fail, due to looking check, * see mptcp_disconnect(). * Attempt it again outside the problematic scope. */ if (!mptcp_disconnect(sk, 0)) { sk->sk_disconnects++; sk->sk_socket->state = SS_UNCONNECTED; } } inet_clear_bit(DEFER_CONNECT, sk); return ret; } static int do_copy_data_nocache(struct sock *sk, int copy, struct iov_iter *from, char *to) { if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) { if (!copy_from_iter_full_nocache(to, copy, from)) return -EFAULT; } else if (!copy_from_iter_full(to, copy, from)) { return -EFAULT; } return 0; } /* open-code sk_stream_memory_free() plus sent limit computation to * avoid indirect calls in fast-path. * Called under the msk socket lock, so we can avoid a bunch of ONCE * annotations. */ static u32 mptcp_send_limit(const struct sock *sk) { const struct mptcp_sock *msk = mptcp_sk(sk); u32 limit, not_sent; if (sk->sk_wmem_queued >= READ_ONCE(sk->sk_sndbuf)) return 0; limit = mptcp_notsent_lowat(sk); if (limit == UINT_MAX) return UINT_MAX; not_sent = msk->write_seq - msk->snd_nxt; if (not_sent >= limit) return 0; return limit - not_sent; } static void mptcp_rps_record_subflows(const struct mptcp_sock *msk) { struct mptcp_subflow_context *subflow; if (!rfs_is_needed()) return; mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); sock_rps_record_flow(ssk); } } static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) { struct mptcp_sock *msk = mptcp_sk(sk); struct page_frag *pfrag; size_t copied = 0; int ret = 0; long timeo; /* silently ignore everything else */ msg->msg_flags &= MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_FASTOPEN; lock_sock(sk); mptcp_rps_record_subflows(msk); if (unlikely(inet_test_bit(DEFER_CONNECT, sk) || msg->msg_flags & MSG_FASTOPEN)) { int copied_syn = 0; ret = mptcp_sendmsg_fastopen(sk, msg, len, &copied_syn); copied += copied_syn; if (ret == -EINPROGRESS && copied_syn > 0) goto out; else if (ret) goto do_error; } timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { ret = sk_stream_wait_connect(sk, &timeo); if (ret) goto do_error; } ret = -EPIPE; if (unlikely(sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))) goto do_error; pfrag = sk_page_frag(sk); while (msg_data_left(msg)) { int total_ts, frag_truesize = 0; struct mptcp_data_frag *dfrag; bool dfrag_collapsed; size_t psize, offset; u32 copy_limit; /* ensure fitting the notsent_lowat() constraint */ copy_limit = mptcp_send_limit(sk); if (!copy_limit) goto wait_for_memory; /* reuse tail pfrag, if possible, or carve a new one from the * page allocator */ dfrag = mptcp_pending_tail(sk); dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); if (!dfrag_collapsed) { if (!mptcp_page_frag_refill(sk, pfrag)) goto wait_for_memory; dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset); frag_truesize = dfrag->overhead; } /* we do not bound vs wspace, to allow a single packet. * memory accounting will prevent execessive memory usage * anyway */ offset = dfrag->offset + dfrag->data_len; psize = pfrag->size - offset; psize = min_t(size_t, psize, msg_data_left(msg)); psize = min_t(size_t, psize, copy_limit); total_ts = psize + frag_truesize; if (!sk_wmem_schedule(sk, total_ts)) goto wait_for_memory; ret = do_copy_data_nocache(sk, psize, &msg->msg_iter, page_address(dfrag->page) + offset); if (ret) goto do_error; /* data successfully copied into the write queue */ sk_forward_alloc_add(sk, -total_ts); copied += psize; dfrag->data_len += psize; frag_truesize += psize; pfrag->offset += frag_truesize; WRITE_ONCE(msk->write_seq, msk->write_seq + psize); /* charge data on mptcp pending queue to the msk socket * Note: we charge such data both to sk and ssk */ sk_wmem_queued_add(sk, frag_truesize); if (!dfrag_collapsed) { get_page(dfrag->page); list_add_tail(&dfrag->list, &msk->rtx_queue); if (!msk->first_pending) msk->first_pending = dfrag; } pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d\n", msk, dfrag->data_seq, dfrag->data_len, dfrag->already_sent, !dfrag_collapsed); continue; wait_for_memory: set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); __mptcp_push_pending(sk, msg->msg_flags); ret = sk_stream_wait_memory(sk, &timeo); if (ret) goto do_error; } if (copied) __mptcp_push_pending(sk, msg->msg_flags); out: release_sock(sk); return copied; do_error: if (copied) goto out; copied = sk_stream_error(sk, msg->msg_flags, ret); goto out; } static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied); static int __mptcp_recvmsg_mskq(struct sock *sk, struct msghdr *msg, size_t len, int flags, int copied_total, struct scm_timestamping_internal *tss, int *cmsg_flags) { struct mptcp_sock *msk = mptcp_sk(sk); struct sk_buff *skb, *tmp; int total_data_len = 0; int copied = 0; skb_queue_walk_safe(&sk->sk_receive_queue, skb, tmp) { u32 delta, offset = MPTCP_SKB_CB(skb)->offset; u32 data_len = skb->len - offset; u32 count; int err; if (flags & MSG_PEEK) { /* skip already peeked skbs */ if (total_data_len + data_len <= copied_total) { total_data_len += data_len; continue; } /* skip the already peeked data in the current skb */ delta = copied_total - total_data_len; offset += delta; data_len -= delta; } count = min_t(size_t, len - copied, data_len); if (!(flags & MSG_TRUNC)) { err = skb_copy_datagram_msg(skb, offset, msg, count); if (unlikely(err < 0)) { if (!copied) return err; break; } } if (MPTCP_SKB_CB(skb)->has_rxtstamp) { tcp_update_recv_tstamps(skb, tss); *cmsg_flags |= MPTCP_CMSG_TS; } copied += count; if (!(flags & MSG_PEEK)) { msk->bytes_consumed += count; if (count < data_len) { MPTCP_SKB_CB(skb)->offset += count; MPTCP_SKB_CB(skb)->map_seq += count; break; } /* avoid the indirect call, we know the destructor is sock_rfree */ skb->destructor = NULL; skb->sk = NULL; atomic_sub(skb->truesize, &sk->sk_rmem_alloc); sk_mem_uncharge(sk, skb->truesize); __skb_unlink(skb, &sk->sk_receive_queue); skb_attempt_defer_free(skb); } if (copied >= len) break; } mptcp_rcv_space_adjust(msk, copied); return copied; } /* receive buffer autotuning. See tcp_rcv_space_adjust for more information. * * Only difference: Use highest rtt estimate of the subflows in use. */ static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) { struct mptcp_subflow_context *subflow; struct sock *sk = (struct sock *)msk; u8 scaling_ratio = U8_MAX; u32 time, advmss = 1; u64 rtt_us, mstamp; msk_owned_by_me(msk); if (copied <= 0) return; if (!msk->rcvspace_init) mptcp_rcv_space_init(msk, msk->first); msk->rcvq_space.copied += copied; mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); rtt_us = msk->rcvq_space.rtt_us; if (rtt_us && time < (rtt_us >> 3)) return; rtt_us = 0; mptcp_for_each_subflow(msk, subflow) { const struct tcp_sock *tp; u64 sf_rtt_us; u32 sf_advmss; tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us); sf_advmss = READ_ONCE(tp->advmss); rtt_us = max(sf_rtt_us, rtt_us); advmss = max(sf_advmss, advmss); scaling_ratio = min(tp->scaling_ratio, scaling_ratio); } msk->rcvq_space.rtt_us = rtt_us; msk->scaling_ratio = scaling_ratio; if (time < (rtt_us >> 3) || rtt_us == 0) return; if (msk->rcvq_space.copied <= msk->rcvq_space.space) goto new_measure; if (mptcp_rcvbuf_grow(sk, msk->rcvq_space.copied)) { /* Make subflows follow along. If we do not do this, we * get drops at subflow level if skbs can't be moved to * the mptcp rx queue fast enough (announced rcv_win can * exceed ssk->sk_rcvbuf). */ mptcp_for_each_subflow(msk, subflow) { struct sock *ssk; bool slow; ssk = mptcp_subflow_tcp_sock(subflow); slow = lock_sock_fast(ssk); /* subflows can be added before tcp_init_transfer() */ if (tcp_sk(ssk)->rcvq_space.space) tcp_rcvbuf_grow(ssk, msk->rcvq_space.copied); unlock_sock_fast(ssk, slow); } } new_measure: msk->rcvq_space.copied = 0; msk->rcvq_space.time = mstamp; } static struct mptcp_subflow_context * __mptcp_first_ready_from(struct mptcp_sock *msk, struct mptcp_subflow_context *subflow) { struct mptcp_subflow_context *start_subflow = subflow; while (!READ_ONCE(subflow->data_avail)) { subflow = mptcp_next_subflow(msk, subflow); if (subflow == start_subflow) return NULL; } return subflow; } static bool __mptcp_move_skbs(struct sock *sk) { struct mptcp_subflow_context *subflow; struct mptcp_sock *msk = mptcp_sk(sk); bool ret = false; if (list_empty(&msk->conn_list)) return false; subflow = list_first_entry(&msk->conn_list, struct mptcp_subflow_context, node); for (;;) { struct sock *ssk; bool slowpath; /* * As an optimization avoid traversing the subflows list * and ev. acquiring the subflow socket lock before baling out */ if (sk_rmem_alloc_get(sk) > sk->sk_rcvbuf) break; subflow = __mptcp_first_ready_from(msk, subflow); if (!subflow) break; ssk = mptcp_subflow_tcp_sock(subflow); slowpath = lock_sock_fast(ssk); ret = __mptcp_move_skbs_from_subflow(msk, ssk) || ret; if (unlikely(ssk->sk_err)) __mptcp_error_report(sk); unlock_sock_fast(ssk, slowpath); subflow = mptcp_next_subflow(msk, subflow); } __mptcp_ofo_queue(msk); if (ret) mptcp_check_data_fin((struct sock *)msk); return ret; } static unsigned int mptcp_inq_hint(const struct sock *sk) { const struct mptcp_sock *msk = mptcp_sk(sk); const struct sk_buff *skb; skb = skb_peek(&sk->sk_receive_queue); if (skb) { u64 hint_val = READ_ONCE(msk->ack_seq) - MPTCP_SKB_CB(skb)->map_seq; if (hint_val >= INT_MAX) return INT_MAX; return (unsigned int)hint_val; } if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) return 1; return 0; } static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags, int *addr_len) { struct mptcp_sock *msk = mptcp_sk(sk); struct scm_timestamping_internal tss; int copied = 0, cmsg_flags = 0; int target; long timeo; /* MSG_ERRQUEUE is really a no-op till we support IP_RECVERR */ if (unlikely(flags & MSG_ERRQUEUE)) return inet_recv_error(sk, msg, len, addr_len); lock_sock(sk); if (unlikely(sk->sk_state == TCP_LISTEN)) { copied = -ENOTCONN; goto out_err; } mptcp_rps_record_subflows(msk); timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); len = min_t(size_t, len, INT_MAX); target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); if (unlikely(msk->recvmsg_inq)) cmsg_flags = MPTCP_CMSG_INQ; while (copied < len) { int err, bytes_read; bytes_read = __mptcp_recvmsg_mskq(sk, msg, len - copied, flags, copied, &tss, &cmsg_flags); if (unlikely(bytes_read < 0)) { if (!copied) copied = bytes_read; goto out_err; } copied += bytes_read; if (skb_queue_empty(&sk->sk_receive_queue) && __mptcp_move_skbs(sk)) continue; /* only the MPTCP socket status is relevant here. The exit * conditions mirror closely tcp_recvmsg() */ if (copied >= target) break; if (copied) { if (sk->sk_err || sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN) || !timeo || signal_pending(current)) break; } else { if (sk->sk_err) { copied = sock_error(sk); break; } if (sk->sk_shutdown & RCV_SHUTDOWN) break; if (sk->sk_state == TCP_CLOSE) { copied = -ENOTCONN; break; } if (!timeo) { copied = -EAGAIN; break; } if (signal_pending(current)) { copied = sock_intr_errno(timeo); break; } } pr_debug("block timeout %ld\n", timeo); mptcp_cleanup_rbuf(msk, copied); err = sk_wait_data(sk, &timeo, NULL); if (err < 0) { err = copied ? : err; goto out_err; } } mptcp_cleanup_rbuf(msk, copied); out_err: if (cmsg_flags && copied >= 0) { if (cmsg_flags & MPTCP_CMSG_TS) tcp_recv_timestamp(msg, sk, &tss); if (cmsg_flags & MPTCP_CMSG_INQ) { unsigned int inq = mptcp_inq_hint(sk); put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq); } } pr_debug("msk=%p rx queue empty=%d copied=%d\n", msk, skb_queue_empty(&sk->sk_receive_queue), copied); release_sock(sk); return copied; } static void mptcp_retransmit_timer(struct timer_list *t) { struct inet_connection_sock *icsk = timer_container_of(icsk, t, icsk_retransmit_timer); struct sock *sk = &icsk->icsk_inet.sk; struct mptcp_sock *msk = mptcp_sk(sk); bh_lock_sock(sk); if (!sock_owned_by_user(sk)) { /* we need a process context to retransmit */ if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags)) mptcp_schedule_work(sk); } else { /* delegate our work to tcp_release_cb() */ __set_bit(MPTCP_RETRANSMIT, &msk->cb_flags); } bh_unlock_sock(sk); sock_put(sk); } static void mptcp_tout_timer(struct timer_list *t) { struct sock *sk = timer_container_of(sk, t, sk_timer); mptcp_schedule_work(sk); sock_put(sk); } /* Find an idle subflow. Return NULL if there is unacked data at tcp * level. * * A backup subflow is returned only if that is the only kind available. */ struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk) { struct sock *backup = NULL, *pick = NULL; struct mptcp_subflow_context *subflow; int min_stale_count = INT_MAX; mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); if (!__mptcp_subflow_active(subflow)) continue; /* still data outstanding at TCP level? skip this */ if (!tcp_rtx_and_write_queues_empty(ssk)) { mptcp_pm_subflow_chk_stale(msk, ssk); min_stale_count = min_t(int, min_stale_count, subflow->stale_count); continue; } if (subflow->backup || subflow->request_bkup) { if (!backup) backup = ssk; continue; } if (!pick) pick = ssk; } if (pick) return pick; /* use backup only if there are no progresses anywhere */ return min_stale_count > 1 ? backup : NULL; } bool __mptcp_retransmit_pending_data(struct sock *sk) { struct mptcp_data_frag *cur, *rtx_head; struct mptcp_sock *msk = mptcp_sk(sk); if (__mptcp_check_fallback(msk)) return false; /* the closing socket has some data untransmitted and/or unacked: * some data in the mptcp rtx queue has not really xmitted yet. * keep it simple and re-inject the whole mptcp level rtx queue */ mptcp_data_lock(sk); __mptcp_clean_una_wakeup(sk); rtx_head = mptcp_rtx_head(sk); if (!rtx_head) { mptcp_data_unlock(sk); return false; } msk->recovery_snd_nxt = msk->snd_nxt; msk->recovery = true; mptcp_data_unlock(sk); msk->first_pending = rtx_head; msk->snd_burst = 0; /* be sure to clear the "sent status" on all re-injected fragments */ list_for_each_entry(cur, &msk->rtx_queue, list) { if (!cur->already_sent) break; cur->already_sent = 0; } return true; } /* flags for __mptcp_close_ssk() */ #define MPTCP_CF_PUSH BIT(1) /* be sure to send a reset only if the caller asked for it, also * clean completely the subflow status when the subflow reaches * TCP_CLOSE state */ static void __mptcp_subflow_disconnect(struct sock *ssk, struct mptcp_subflow_context *subflow, unsigned int flags) { if (((1 << ssk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) || subflow->send_fastclose) { /* The MPTCP code never wait on the subflow sockets, TCP-level * disconnect should never fail */ WARN_ON_ONCE(tcp_disconnect(ssk, 0)); mptcp_subflow_ctx_reset(subflow); } else { tcp_shutdown(ssk, SEND_SHUTDOWN); } } /* subflow sockets can be either outgoing (connect) or incoming * (accept). * * Outgoing subflows use in-kernel sockets. * Incoming subflows do not have their own 'struct socket' allocated, * so we need to use tcp_close() after detaching them from the mptcp * parent socket. */ static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, struct mptcp_subflow_context *subflow, unsigned int flags) { struct mptcp_sock *msk = mptcp_sk(sk); bool dispose_it, need_push = false; /* If the first subflow moved to a close state before accept, e.g. due * to an incoming reset or listener shutdown, the subflow socket is * already deleted by inet_child_forget() and the mptcp socket can't * survive too. */ if (msk->in_accept_queue && msk->first == ssk && (sock_flag(sk, SOCK_DEAD) || sock_flag(ssk, SOCK_DEAD))) { /* ensure later check in mptcp_worker() will dispose the msk */ sock_set_flag(sk, SOCK_DEAD); mptcp_set_close_tout(sk, tcp_jiffies32 - (mptcp_close_timeout(sk) + 1)); lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); mptcp_subflow_drop_ctx(ssk); goto out_release; } dispose_it = msk->free_first || ssk != msk->first; if (dispose_it) list_del(&subflow->node); lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); if (subflow->send_fastclose && ssk->sk_state != TCP_CLOSE) tcp_set_state(ssk, TCP_CLOSE); need_push = (flags & MPTCP_CF_PUSH) && __mptcp_retransmit_pending_data(sk); if (!dispose_it) { __mptcp_subflow_disconnect(ssk, subflow, flags); release_sock(ssk); goto out; } subflow->disposable = 1; /* if ssk hit tcp_done(), tcp_cleanup_ulp() cleared the related ops * the ssk has been already destroyed, we just need to release the * reference owned by msk; */ if (!inet_csk(ssk)->icsk_ulp_ops) { WARN_ON_ONCE(!sock_flag(ssk, SOCK_DEAD)); kfree_rcu(subflow, rcu); } else { /* otherwise tcp will dispose of the ssk and subflow ctx */ __tcp_close(ssk, 0); /* close acquired an extra ref */ __sock_put(ssk); } out_release: __mptcp_subflow_error_report(sk, ssk); release_sock(ssk); sock_put(ssk); if (ssk == msk->first) WRITE_ONCE(msk->first, NULL); out: __mptcp_sync_sndbuf(sk); if (need_push) __mptcp_push_pending(sk, 0); /* Catch every 'all subflows closed' scenario, including peers silently * closing them, e.g. due to timeout. * For established sockets, allow an additional timeout before closing, * as the protocol can still create more subflows. */ if (list_is_singular(&msk->conn_list) && msk->first && inet_sk_state_load(msk->first) == TCP_CLOSE) { if (sk->sk_state != TCP_ESTABLISHED || msk->in_accept_queue || sock_flag(sk, SOCK_DEAD)) { mptcp_set_state(sk, TCP_CLOSE); mptcp_close_wake_up(sk); } else { mptcp_start_tout_timer(sk); } } } void mptcp_close_ssk(struct sock *sk, struct sock *ssk, struct mptcp_subflow_context *subflow) { /* The first subflow can already be closed and still in the list */ if (subflow->close_event_done) return; subflow->close_event_done = true; if (sk->sk_state == TCP_ESTABLISHED) mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL); /* subflow aborted before reaching the fully_established status * attempt the creation of the next subflow */ mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow); __mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH); } static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) { return 0; } static void __mptcp_close_subflow(struct sock *sk) { struct mptcp_subflow_context *subflow, *tmp; struct mptcp_sock *msk = mptcp_sk(sk); might_sleep(); mptcp_for_each_subflow_safe(msk, subflow, tmp) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); int ssk_state = inet_sk_state_load(ssk); if (ssk_state != TCP_CLOSE && (ssk_state != TCP_CLOSE_WAIT || inet_sk_state_load(sk) != TCP_ESTABLISHED || __mptcp_check_fallback(msk))) continue; /* 'subflow_data_ready' will re-sched once rx queue is empty */ if (!skb_queue_empty_lockless(&ssk->sk_receive_queue)) continue; mptcp_close_ssk(sk, ssk, subflow); } } static bool mptcp_close_tout_expired(const struct sock *sk) { if (!inet_csk(sk)->icsk_mtup.probe_timestamp || sk->sk_state == TCP_CLOSE) return false; return time_after32(tcp_jiffies32, inet_csk(sk)->icsk_mtup.probe_timestamp + mptcp_close_timeout(sk)); } static void mptcp_check_fastclose(struct mptcp_sock *msk) { struct mptcp_subflow_context *subflow, *tmp; struct sock *sk = (struct sock *)msk; if (likely(!READ_ONCE(msk->rcv_fastclose))) return; mptcp_token_destroy(msk); mptcp_for_each_subflow_safe(msk, subflow, tmp) { struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); bool slow; slow = lock_sock_fast(tcp_sk); if (tcp_sk->sk_state != TCP_CLOSE) { mptcp_send_active_reset_reason(tcp_sk); tcp_set_state(tcp_sk, TCP_CLOSE); } unlock_sock_fast(tcp_sk, slow); } /* Mirror the tcp_reset() error propagation */ switch (sk->sk_state) { case TCP_SYN_SENT: WRITE_ONCE(sk->sk_err, ECONNREFUSED); break; case TCP_CLOSE_WAIT: WRITE_ONCE(sk->sk_err, EPIPE); break; case TCP_CLOSE: return; default: WRITE_ONCE(sk->sk_err, ECONNRESET); } mptcp_set_state(sk, TCP_CLOSE); WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags); /* the calling mptcp_worker will properly destroy the socket */ if (sock_flag(sk, SOCK_DEAD)) return; sk->sk_state_change(sk); sk_error_report(sk); } static void __mptcp_retrans(struct sock *sk) { struct mptcp_sendmsg_info info = { .data_lock_held = true, }; struct mptcp_sock *msk = mptcp_sk(sk); struct mptcp_subflow_context *subflow; struct mptcp_data_frag *dfrag; struct sock *ssk; int ret, err; u16 len = 0; mptcp_clean_una_wakeup(sk); /* first check ssk: need to kick "stale" logic */ err = mptcp_sched_get_retrans(msk); dfrag = mptcp_rtx_head(sk); if (!dfrag) { if (mptcp_data_fin_enabled(msk)) { struct inet_connection_sock *icsk = inet_csk(sk); WRITE_ONCE(icsk->icsk_retransmits, icsk->icsk_retransmits + 1); mptcp_set_datafin_timeout(sk); mptcp_send_ack(msk); goto reset_timer; } if (!mptcp_send_head(sk)) return; goto reset_timer; } if (err) goto reset_timer; mptcp_for_each_subflow(msk, subflow) { if (READ_ONCE(subflow->scheduled)) { u16 copied = 0; mptcp_subflow_set_scheduled(subflow, false); ssk = mptcp_subflow_tcp_sock(subflow); lock_sock(ssk); /* limit retransmission to the bytes already sent on some subflows */ info.sent = 0; info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : dfrag->already_sent; /* * make the whole retrans decision, xmit, disallow * fallback atomic */ spin_lock_bh(&msk->fallback_lock); if (__mptcp_check_fallback(msk)) { spin_unlock_bh(&msk->fallback_lock); release_sock(ssk); return; } while (info.sent < info.limit) { ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); if (ret <= 0) break; MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS); copied += ret; info.sent += ret; } if (copied) { len = max(copied, len); tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, info.size_goal); msk->allow_infinite_fallback = false; } spin_unlock_bh(&msk->fallback_lock); release_sock(ssk); } } msk->bytes_retrans += len; dfrag->already_sent = max(dfrag->already_sent, len); reset_timer: mptcp_check_and_set_pending(sk); if (!mptcp_rtx_timer_pending(sk)) mptcp_reset_rtx_timer(sk); } /* schedule the timeout timer for the relevant event: either close timeout * or mp_fail timeout. The close timeout takes precedence on the mp_fail one */ void mptcp_reset_tout_timer(struct mptcp_sock *msk, unsigned long fail_tout) { struct sock *sk = (struct sock *)msk; unsigned long timeout, close_timeout; if (!fail_tout && !inet_csk(sk)->icsk_mtup.probe_timestamp) return; close_timeout = (unsigned long)inet_csk(sk)->icsk_mtup.probe_timestamp - tcp_jiffies32 + jiffies + mptcp_close_timeout(sk); /* the close timeout takes precedence on the fail one, and here at least one of * them is active */ timeout = inet_csk(sk)->icsk_mtup.probe_timestamp ? close_timeout : fail_tout; sk_reset_timer(sk, &sk->sk_timer, timeout); } static void mptcp_mp_fail_no_response(struct mptcp_sock *msk) { struct sock *ssk = msk->first; bool slow; if (!ssk) return; pr_debug("MP_FAIL doesn't respond, reset the subflow\n"); slow = lock_sock_fast(ssk); mptcp_subflow_reset(ssk); WRITE_ONCE(mptcp_subflow_ctx(ssk)->fail_tout, 0); unlock_sock_fast(ssk, slow); } static void mptcp_do_fastclose(struct sock *sk) { struct mptcp_subflow_context *subflow, *tmp; struct mptcp_sock *msk = mptcp_sk(sk); mptcp_set_state(sk, TCP_CLOSE); /* Explicitly send the fastclose reset as need */ if (__mptcp_check_fallback(msk)) return; mptcp_for_each_subflow_safe(msk, subflow, tmp) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); lock_sock(ssk); /* Some subflow socket states don't allow/need a reset.*/ if ((1 << ssk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) goto unlock; subflow->send_fastclose = 1; tcp_send_active_reset(ssk, ssk->sk_allocation, SK_RST_REASON_TCP_ABORT_ON_CLOSE); unlock: release_sock(ssk); } } static void mptcp_worker(struct work_struct *work) { struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); struct sock *sk = (struct sock *)msk; unsigned long fail_tout; int state; lock_sock(sk); state = sk->sk_state; if (unlikely((1 << state) & (TCPF_CLOSE | TCPF_LISTEN))) goto unlock; mptcp_check_fastclose(msk); mptcp_pm_worker(msk); mptcp_check_send_data_fin(sk); mptcp_check_data_fin_ack(sk); mptcp_check_data_fin(sk); if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) __mptcp_close_subflow(sk); if (mptcp_close_tout_expired(sk)) { struct mptcp_subflow_context *subflow, *tmp; mptcp_do_fastclose(sk); mptcp_for_each_subflow_safe(msk, subflow, tmp) __mptcp_close_ssk(sk, subflow->tcp_sock, subflow, 0); mptcp_close_wake_up(sk); } if (sock_flag(sk, SOCK_DEAD) && sk->sk_state == TCP_CLOSE) { __mptcp_destroy_sock(sk); goto unlock; } if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) __mptcp_retrans(sk); fail_tout = msk->first ? READ_ONCE(mptcp_subflow_ctx(msk->first)->fail_tout) : 0; if (fail_tout && time_after(jiffies, fail_tout)) mptcp_mp_fail_no_response(msk); unlock: release_sock(sk); sock_put(sk); } static void __mptcp_init_sock(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); INIT_LIST_HEAD(&msk->conn_list); INIT_LIST_HEAD(&msk->join_list); INIT_LIST_HEAD(&msk->rtx_queue); INIT_WORK(&msk->work, mptcp_worker); msk->out_of_order_queue = RB_ROOT; msk->first_pending = NULL; msk->timer_ival = TCP_RTO_MIN; msk->scaling_ratio = TCP_DEFAULT_SCALING_RATIO; WRITE_ONCE(msk->first, NULL); inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); msk->allow_infinite_fallback = true; msk->allow_subflows = true; msk->recovery = false; msk->subflow_id = 1; msk->last_data_sent = tcp_jiffies32; msk->last_data_recv = tcp_jiffies32; msk->last_ack_recv = tcp_jiffies32; mptcp_pm_data_init(msk); spin_lock_init(&msk->fallback_lock); /* re-use the csk retrans timer for MPTCP-level retrans */ timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); timer_setup(&sk->sk_timer, mptcp_tout_timer, 0); } static void mptcp_ca_reset(struct sock *sk) { struct inet_connection_sock *icsk = inet_csk(sk); tcp_assign_congestion_control(sk); strscpy(mptcp_sk(sk)->ca_name, icsk->icsk_ca_ops->name, sizeof(mptcp_sk(sk)->ca_name)); /* no need to keep a reference to the ops, the name will suffice */ tcp_cleanup_congestion_control(sk); icsk->icsk_ca_ops = NULL; } static int mptcp_init_sock(struct sock *sk) { struct net *net = sock_net(sk); int ret; __mptcp_init_sock(sk); if (!mptcp_is_enabled(net)) return -ENOPROTOOPT; if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net)) return -ENOMEM; rcu_read_lock(); ret = mptcp_init_sched(mptcp_sk(sk), mptcp_sched_find(mptcp_get_scheduler(net))); rcu_read_unlock(); if (ret) return ret; set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags); /* fetch the ca name; do it outside __mptcp_init_sock(), so that clone will * propagate the correct value */ mptcp_ca_reset(sk); sk_sockets_allocated_inc(sk); sk->sk_rcvbuf = READ_ONCE(net->ipv4.sysctl_tcp_rmem[1]); sk->sk_sndbuf = READ_ONCE(net->ipv4.sysctl_tcp_wmem[1]); return 0; } static void __mptcp_clear_xmit(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); struct mptcp_data_frag *dtmp, *dfrag; msk->first_pending = NULL; list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) dfrag_clear(sk, dfrag); } void mptcp_cancel_work(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); if (cancel_work_sync(&msk->work)) __sock_put(sk); } void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) { lock_sock(ssk); switch (ssk->sk_state) { case TCP_LISTEN: if (!(how & RCV_SHUTDOWN)) break; fallthrough; case TCP_SYN_SENT: WARN_ON_ONCE(tcp_disconnect(ssk, O_NONBLOCK)); break; default: if (__mptcp_check_fallback(mptcp_sk(sk))) { pr_debug("Fallback\n"); ssk->sk_shutdown |= how; tcp_shutdown(ssk, how); /* simulate the data_fin ack reception to let the state * machine move forward */ WRITE_ONCE(mptcp_sk(sk)->snd_una, mptcp_sk(sk)->snd_nxt); mptcp_schedule_work(sk); } else { pr_debug("Sending DATA_FIN on subflow %p\n", ssk); tcp_send_ack(ssk); if (!mptcp_rtx_timer_pending(sk)) mptcp_reset_rtx_timer(sk); } break; } release_sock(ssk); } void mptcp_set_state(struct sock *sk, int state) { int oldstate = sk->sk_state; switch (state) { case TCP_ESTABLISHED: if (oldstate != TCP_ESTABLISHED) MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB); break; case TCP_CLOSE_WAIT: /* Unlike TCP, MPTCP sk would not have the TCP_SYN_RECV state: * MPTCP "accepted" sockets will be created later on. So no * transition from TCP_SYN_RECV to TCP_CLOSE_WAIT. */ break; default: if (oldstate == TCP_ESTABLISHED || oldstate == TCP_CLOSE_WAIT) MPTCP_DEC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB); } inet_sk_state_store(sk, state); } static const unsigned char new_state[16] = { /* current state: new state: action: */ [0 /* (Invalid) */] = TCP_CLOSE, [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, [TCP_SYN_SENT] = TCP_CLOSE, [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, [TCP_TIME_WAIT] = TCP_CLOSE, /* should not happen ! */ [TCP_CLOSE] = TCP_CLOSE, [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, [TCP_LAST_ACK] = TCP_LAST_ACK, [TCP_LISTEN] = TCP_CLOSE, [TCP_CLOSING] = TCP_CLOSING, [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ }; static int mptcp_close_state(struct sock *sk) { int next = (int)new_state[sk->sk_state]; int ns = next & TCP_STATE_MASK; mptcp_set_state(sk, ns); return next & TCP_ACTION_FIN; } static void mptcp_check_send_data_fin(struct sock *sk) { struct mptcp_subflow_context *subflow; struct mptcp_sock *msk = mptcp_sk(sk); pr_debug("msk=%p snd_data_fin_enable=%d pending=%d snd_nxt=%llu write_seq=%llu\n", msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk), msk->snd_nxt, msk->write_seq); /* we still need to enqueue subflows or not really shutting down, * skip this */ if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq || mptcp_send_head(sk)) return; WRITE_ONCE(msk->snd_nxt, msk->write_seq); mptcp_for_each_subflow(msk, subflow) { struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN); } } static void __mptcp_wr_shutdown(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); pr_debug("msk=%p snd_data_fin_enable=%d shutdown=%x state=%d pending=%d\n", msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state, !!mptcp_send_head(sk)); /* will be ignored by fallback sockets */ WRITE_ONCE(msk->write_seq, msk->write_seq + 1); WRITE_ONCE(msk->snd_data_fin_enable, 1); mptcp_check_send_data_fin(sk); } static void __mptcp_destroy_sock(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); pr_debug("msk=%p\n", msk); might_sleep(); mptcp_stop_rtx_timer(sk); sk_stop_timer(sk, &sk->sk_timer); msk->pm.status = 0; mptcp_release_sched(msk); sk->sk_prot->destroy(sk); sk_stream_kill_queues(sk); xfrm_sk_free_policy(sk); sock_put(sk); } void __mptcp_unaccepted_force_close(struct sock *sk) { sock_set_flag(sk, SOCK_DEAD); mptcp_do_fastclose(sk); __mptcp_destroy_sock(sk); } static __poll_t mptcp_check_readable(struct sock *sk) { return mptcp_epollin_ready(sk) ? EPOLLIN | EPOLLRDNORM : 0; } static void mptcp_check_listen_stop(struct sock *sk) { struct sock *ssk; if (inet_sk_state_load(sk) != TCP_LISTEN) return; sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); ssk = mptcp_sk(sk)->first; if (WARN_ON_ONCE(!ssk || inet_sk_state_load(ssk) != TCP_LISTEN)) return; lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); tcp_set_state(ssk, TCP_CLOSE); mptcp_subflow_queue_clean(sk, ssk); inet_csk_listen_stop(ssk); mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CLOSED); release_sock(ssk); } bool __mptcp_close(struct sock *sk, long timeout) { struct mptcp_subflow_context *subflow; struct mptcp_sock *msk = mptcp_sk(sk); bool do_cancel_work = false; int subflows_alive = 0; WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) { mptcp_check_listen_stop(sk); mptcp_set_state(sk, TCP_CLOSE); goto cleanup; } if (mptcp_data_avail(msk) || timeout < 0) { /* If the msk has read data, or the caller explicitly ask it, * do the MPTCP equivalent of TCP reset, aka MPTCP fastclose */ mptcp_do_fastclose(sk); timeout = 0; } else if (mptcp_close_state(sk)) { __mptcp_wr_shutdown(sk); } sk_stream_wait_close(sk, timeout); cleanup: /* orphan all the subflows */ mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); bool slow = lock_sock_fast_nested(ssk); subflows_alive += ssk->sk_state != TCP_CLOSE; /* since the close timeout takes precedence on the fail one, * cancel the latter */ if (ssk == msk->first) subflow->fail_tout = 0; /* detach from the parent socket, but allow data_ready to * push incoming data into the mptcp stack, to properly ack it */ ssk->sk_socket = NULL; ssk->sk_wq = NULL; unlock_sock_fast(ssk, slow); } sock_orphan(sk); /* all the subflows are closed, only timeout can change the msk * state, let's not keep resources busy for no reasons */ if (subflows_alive == 0) mptcp_set_state(sk, TCP_CLOSE); sock_hold(sk); pr_debug("msk=%p state=%d\n", sk, sk->sk_state); mptcp_pm_connection_closed(msk); if (sk->sk_state == TCP_CLOSE) { __mptcp_destroy_sock(sk); do_cancel_work = true; } else { mptcp_start_tout_timer(sk); } return do_cancel_work; } static void mptcp_close(struct sock *sk, long timeout) { bool do_cancel_work; lock_sock(sk); do_cancel_work = __mptcp_close(sk, timeout); release_sock(sk); if (do_cancel_work) mptcp_cancel_work(sk); sock_put(sk); } static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) { #if IS_ENABLED(CONFIG_MPTCP_IPV6) const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); struct ipv6_pinfo *msk6 = inet6_sk(msk); msk->sk_v6_daddr = ssk->sk_v6_daddr; msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; if (msk6 && ssk6) { msk6->saddr = ssk6->saddr; msk6->flow_label = ssk6->flow_label; } #endif inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; } static int mptcp_disconnect(struct sock *sk, int flags) { struct mptcp_sock *msk = mptcp_sk(sk); /* We are on the fastopen error path. We can't call straight into the * subflows cleanup code due to lock nesting (we are already under * msk->firstsocket lock). */ if (msk->fastopening) return -EBUSY; mptcp_check_listen_stop(sk); mptcp_set_state(sk, TCP_CLOSE); mptcp_stop_rtx_timer(sk); mptcp_stop_tout_timer(sk); mptcp_pm_connection_closed(msk); /* msk->subflow is still intact, the following will not free the first * subflow */ mptcp_do_fastclose(sk); mptcp_destroy_common(msk); /* The first subflow is already in TCP_CLOSE status, the following * can't overlap with a fallback anymore */ spin_lock_bh(&msk->fallback_lock); msk->allow_subflows = true; msk->allow_infinite_fallback = true; WRITE_ONCE(msk->flags, 0); spin_unlock_bh(&msk->fallback_lock); msk->cb_flags = 0; msk->recovery = false; WRITE_ONCE(msk->can_ack, false); WRITE_ONCE(msk->fully_established, false); WRITE_ONCE(msk->rcv_data_fin, false); WRITE_ONCE(msk->snd_data_fin_enable, false); WRITE_ONCE(msk->rcv_fastclose, false); WRITE_ONCE(msk->use_64bit_ack, false); WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); mptcp_pm_data_reset(msk); mptcp_ca_reset(sk); msk->bytes_consumed = 0; msk->bytes_acked = 0; msk->bytes_received = 0; msk->bytes_sent = 0; msk->bytes_retrans = 0; msk->rcvspace_init = 0; WRITE_ONCE(sk->sk_shutdown, 0); sk_error_report(sk); return 0; } #if IS_ENABLED(CONFIG_MPTCP_IPV6) static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) { struct mptcp6_sock *msk6 = container_of(mptcp_sk(sk), struct mptcp6_sock, msk); return &msk6->np; } static void mptcp_copy_ip6_options(struct sock *newsk, const struct sock *sk) { const struct ipv6_pinfo *np = inet6_sk(sk); struct ipv6_txoptions *opt; struct ipv6_pinfo *newnp; newnp = inet6_sk(newsk); rcu_read_lock(); opt = rcu_dereference(np->opt); if (opt) { opt = ipv6_dup_options(newsk, opt); if (!opt) net_warn_ratelimited("%s: Failed to copy ip6 options\n", __func__); } RCU_INIT_POINTER(newnp->opt, opt); rcu_read_unlock(); } #endif static void mptcp_copy_ip_options(struct sock *newsk, const struct sock *sk) { struct ip_options_rcu *inet_opt, *newopt = NULL; const struct inet_sock *inet = inet_sk(sk); struct inet_sock *newinet; newinet = inet_sk(newsk); rcu_read_lock(); inet_opt = rcu_dereference(inet->inet_opt); if (inet_opt) { newopt = sock_kmemdup(newsk, inet_opt, sizeof(*inet_opt) + inet_opt->opt.optlen, GFP_ATOMIC); if (!newopt) net_warn_ratelimited("%s: Failed to copy ip options\n", __func__); } RCU_INIT_POINTER(newinet->inet_opt, newopt); rcu_read_unlock(); } struct sock *mptcp_sk_clone_init(const struct sock *sk, const struct mptcp_options_received *mp_opt, struct sock *ssk, struct request_sock *req) { struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); struct mptcp_subflow_context *subflow; struct mptcp_sock *msk; if (!nsk) return NULL; #if IS_ENABLED(CONFIG_MPTCP_IPV6) if (nsk->sk_family == AF_INET6) inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); #endif __mptcp_init_sock(nsk); #if IS_ENABLED(CONFIG_MPTCP_IPV6) if (nsk->sk_family == AF_INET6) mptcp_copy_ip6_options(nsk, sk); else #endif mptcp_copy_ip_options(nsk, sk); msk = mptcp_sk(nsk); WRITE_ONCE(msk->local_key, subflow_req->local_key); WRITE_ONCE(msk->token, subflow_req->token); msk->in_accept_queue = 1; WRITE_ONCE(msk->fully_established, false); if (mp_opt->suboptions & OPTION_MPTCP_CSUMREQD) WRITE_ONCE(msk->csum_enabled, true); WRITE_ONCE(msk->write_seq, subflow_req->idsn + 1); WRITE_ONCE(msk->snd_nxt, msk->write_seq); WRITE_ONCE(msk->snd_una, msk->write_seq); WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd); msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq; mptcp_init_sched(msk, mptcp_sk(sk)->sched); /* passive msk is created after the first/MPC subflow */ msk->subflow_id = 2; sock_reset_flag(nsk, SOCK_RCU_FREE); security_inet_csk_clone(nsk, req); /* this can't race with mptcp_close(), as the msk is * not yet exposted to user-space */ mptcp_set_state(nsk, TCP_ESTABLISHED); /* The msk maintain a ref to each subflow in the connections list */ WRITE_ONCE(msk->first, ssk); subflow = mptcp_subflow_ctx(ssk); list_add(&subflow->node, &msk->conn_list); sock_hold(ssk); /* new mpc subflow takes ownership of the newly * created mptcp socket */ mptcp_token_accept(subflow_req, msk); /* set msk addresses early to ensure mptcp_pm_get_local_id() * uses the correct data */ mptcp_copy_inaddrs(nsk, ssk); __mptcp_propagate_sndbuf(nsk, ssk); mptcp_rcv_space_init(msk, ssk); if (mp_opt->suboptions & OPTION_MPTCP_MPC_ACK) __mptcp_subflow_fully_established(msk, subflow, mp_opt); bh_unlock_sock(nsk); /* note: the newly allocated socket refcount is 2 now */ return nsk; } void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) { const struct tcp_sock *tp = tcp_sk(ssk); msk->rcvspace_init = 1; msk->rcvq_space.copied = 0; msk->rcvq_space.rtt_us = 0; msk->rcvq_space.time = tp->tcp_mstamp; /* initial rcv_space offering made to peer */ msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, TCP_INIT_CWND * tp->advmss); if (msk->rcvq_space.space == 0) msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; } void mptcp_destroy_common(struct mptcp_sock *msk) { struct mptcp_subflow_context *subflow, *tmp; struct sock *sk = (struct sock *)msk; __mptcp_clear_xmit(sk); /* join list will be eventually flushed (with rst) at sock lock release time */ mptcp_for_each_subflow_safe(msk, subflow, tmp) __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, 0); __skb_queue_purge(&sk->sk_receive_queue); skb_rbtree_purge(&msk->out_of_order_queue); /* move all the rx fwd alloc into the sk_mem_reclaim_final in * inet_sock_destruct() will dispose it */ mptcp_token_destroy(msk); mptcp_pm_destroy(msk); } static void mptcp_destroy(struct sock *sk) { struct mptcp_sock *msk = mptcp_sk(sk); /* allow the following to close even the initial subflow */ msk->free_first = 1; mptcp_destroy_common(msk); sk_sockets_allocated_dec(sk); } void __mptcp_data_acked(struct sock *sk) { if (!sock_owned_by_user(sk)) __mptcp_clean_una(sk); else __set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->cb_flags); } void __mptcp_check_push(struct sock *sk, struct sock *ssk) { if (!sock_owned_by_user(sk)) __mptcp_subflow_push_pending(sk, ssk, false); else __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); } #define MPTCP_FLAGS_PROCESS_CTX_NEED (BIT(MPTCP_PUSH_PENDING) | \ BIT(MPTCP_RETRANSMIT) | \ BIT(MPTCP_FLUSH_JOIN_LIST) | \ BIT(MPTCP_DEQUEUE)) /* processes deferred events and flush wmem */ static void mptcp_release_cb(struct sock *sk) __must_hold(&sk->sk_lock.slock) { struct mptcp_sock *msk = mptcp_sk(sk); for (;;) { unsigned long flags = (msk->cb_flags & MPTCP_FLAGS_PROCESS_CTX_NEED); struct list_head join_list; if (!flags) break; INIT_LIST_HEAD(&join_list); list_splice_init(&msk->join_list, &join_list); /* the following actions acquire the subflow socket lock * * 1) can't be invoked in atomic scope * 2) must avoid ABBA deadlock with msk socket spinlock: the RX * datapath acquires the msk socket spinlock while helding * the subflow socket lock */ msk->cb_flags &= ~flags; spin_unlock_bh(&sk->sk_lock.slock); if (flags & BIT(MPTCP_FLUSH_JOIN_LIST)) __mptcp_flush_join_list(sk, &join_list); if (flags & BIT(MPTCP_PUSH_PENDING)) __mptcp_push_pending(sk, 0); if (flags & BIT(MPTCP_RETRANSMIT)) __mptcp_retrans(sk); if ((flags & BIT(MPTCP_DEQUEUE)) && __mptcp_move_skbs(sk)) { /* notify ack seq update */ mptcp_cleanup_rbuf(msk, 0); sk->sk_data_ready(sk); } cond_resched(); spin_lock_bh(&sk->sk_lock.slock); } if (__test_and_clear_bit(MPTCP_CLEAN_UNA, &msk->cb_flags)) __mptcp_clean_una_wakeup(sk); if (unlikely(msk->cb_flags)) { /* be sure to sync the msk state before taking actions * depending on sk_state (MPTCP_ERROR_REPORT) * On sk release avoid actions depending on the first subflow */ if (__test_and_clear_bit(MPTCP_SYNC_STATE, &msk->cb_flags) && msk->first) __mptcp_sync_state(sk, msk->pending_state); if (__test_and_clear_bit(MPTCP_ERROR_REPORT, &msk->cb_flags)) __mptcp_error_report(sk); if (__test_and_clear_bit(MPTCP_SYNC_SNDBUF, &msk->cb_flags)) __mptcp_sync_sndbuf(sk); } } /* MP_JOIN client subflow must wait for 4th ack before sending any data: * TCP can't schedule delack timer before the subflow is fully established. * MPTCP uses the delack timer to do 3rd ack retransmissions */ static void schedule_3rdack_retransmission(struct sock *ssk) { struct inet_connection_sock *icsk = inet_csk(ssk); struct tcp_sock *tp = tcp_sk(ssk); unsigned long timeout; if (READ_ONCE(mptcp_subflow_ctx(ssk)->fully_established)) return; /* reschedule with a timeout above RTT, as we must look only for drop */ if (tp->srtt_us) timeout = usecs_to_jiffies(tp->srtt_us >> (3 - 1)); else timeout = TCP_TIMEOUT_INIT; timeout += jiffies; WARN_ON_ONCE(icsk->icsk_ack.pending & ICSK_ACK_TIMER); smp_store_release(&icsk->icsk_ack.pending, icsk->icsk_ack.pending | ICSK_ACK_SCHED | ICSK_ACK_TIMER); sk_reset_timer(ssk, &icsk->icsk_delack_timer, timeout); } void mptcp_subflow_process_delegated(struct sock *ssk, long status) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); struct sock *sk = subflow->conn; if (status & BIT(MPTCP_DELEGATE_SEND)) { mptcp_data_lock(sk); if (!sock_owned_by_user(sk)) __mptcp_subflow_push_pending(sk, ssk, true); else __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); mptcp_data_unlock(sk); } if (status & BIT(MPTCP_DELEGATE_SNDBUF)) { mptcp_data_lock(sk); if (!sock_owned_by_user(sk)) __mptcp_sync_sndbuf(sk); else __set_bit(MPTCP_SYNC_SNDBUF, &mptcp_sk(sk)->cb_flags); mptcp_data_unlock(sk); } if (status & BIT(MPTCP_DELEGATE_ACK)) schedule_3rdack_retransmission(ssk); } static int mptcp_hash(struct sock *sk) { /* should never be called, * we hash the TCP subflows not the MPTCP socket */ WARN_ON_ONCE(1); return 0; } static void mptcp_unhash(struct sock *sk) { /* called from sk_common_release(), but nothing to do here */ } static int mptcp_get_port(struct sock *sk, unsigned short snum) { struct mptcp_sock *msk = mptcp_sk(sk); pr_debug("msk=%p, ssk=%p\n", msk, msk->first); if (WARN_ON_ONCE(!msk->first)) return -EINVAL; return inet_csk_get_port(msk->first, snum); } void mptcp_finish_connect(struct sock *ssk) { struct mptcp_subflow_context *subflow; struct mptcp_sock *msk; struct sock *sk; subflow = mptcp_subflow_ctx(ssk); sk = subflow->conn; msk = mptcp_sk(sk); pr_debug("msk=%p, token=%u\n", sk, subflow->token); subflow->map_seq = subflow->iasn; subflow->map_subflow_seq = 1; /* the socket is not connected yet, no msk/subflow ops can access/race * accessing the field below */ WRITE_ONCE(msk->local_key, subflow->local_key); mptcp_pm_new_connection(msk, ssk, 0); } void mptcp_sock_graft(struct sock *sk, struct socket *parent) { write_lock_bh(&sk->sk_callback_lock); rcu_assign_pointer(sk->sk_wq, &parent->wq); sk_set_socket(sk, parent); write_unlock_bh(&sk->sk_callback_lock); } bool mptcp_finish_join(struct sock *ssk) { struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); struct mptcp_sock *msk = mptcp_sk(subflow->conn); struct sock *parent = (void *)msk; bool ret = true; pr_debug("msk=%p, subflow=%p\n", msk, subflow); /* mptcp socket already closing? */ if (!mptcp_is_fully_established(parent)) { subflow->reset_reason = MPTCP_RST_EMPTCP; return false; } /* active subflow, already present inside the conn_list */ if (!list_empty(&subflow->node)) { spin_lock_bh(&msk->fallback_lock); if (!msk->allow_subflows) { spin_unlock_bh(&msk->fallback_lock); return false; } mptcp_subflow_joined(msk, ssk); spin_unlock_bh(&msk->fallback_lock); mptcp_propagate_sndbuf(parent, ssk); return true; } if (!mptcp_pm_allow_new_subflow(msk)) { MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_JOINREJECTED); goto err_prohibited; } /* If we can't acquire msk socket lock here, let the release callback * handle it */ mptcp_data_lock(parent); if (!sock_owned_by_user(parent)) { ret = __mptcp_finish_join(msk, ssk); if (ret) { sock_hold(ssk); list_add_tail(&subflow->node, &msk->conn_list); } } else { sock_hold(ssk); list_add_tail(&subflow->node, &msk->join_list); __set_bit(MPTCP_FLUSH_JOIN_LIST, &msk->cb_flags); } mptcp_data_unlock(parent); if (!ret) { err_prohibited: subflow->reset_reason = MPTCP_RST_EPROHIBIT; return false; } return true; } static void mptcp_shutdown(struct sock *sk, int how) { pr_debug("sk=%p, how=%d\n", sk, how); if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk)) __mptcp_wr_shutdown(sk); } static int mptcp_ioctl_outq(const struct mptcp_sock *msk, u64 v) { const struct sock *sk = (void *)msk; u64 delta; if (sk->sk_state == TCP_LISTEN) return -EINVAL; if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) return 0; delta = msk->write_seq - v; if (__mptcp_check_fallback(msk) && msk->first) { struct tcp_sock *tp = tcp_sk(msk->first); /* the first subflow is disconnected after close - see * __mptcp_close_ssk(). tcp_disconnect() moves the write_seq * so ignore that status, too. */ if (!((1 << msk->first->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE))) delta += READ_ONCE(tp->write_seq) - tp->snd_una; } if (delta > INT_MAX) delta = INT_MAX; return (int)delta; } static int mptcp_ioctl(struct sock *sk, int cmd, int *karg) { struct mptcp_sock *msk = mptcp_sk(sk); bool slow; switch (cmd) { case SIOCINQ: if (sk->sk_state == TCP_LISTEN) return -EINVAL; lock_sock(sk); if (__mptcp_move_skbs(sk)) mptcp_cleanup_rbuf(msk, 0); *karg = mptcp_inq_hint(sk); release_sock(sk); break; case SIOCOUTQ: slow = lock_sock_fast(sk); *karg = mptcp_ioctl_outq(msk, READ_ONCE(msk->snd_una)); unlock_sock_fast(sk, slow); break; case SIOCOUTQNSD: slow = lock_sock_fast(sk); *karg = mptcp_ioctl_outq(msk, msk->snd_nxt); unlock_sock_fast(sk, slow); break; default: return -ENOIOCTLCMD; } return 0; } static int mptcp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) { struct mptcp_subflow_context *subflow; struct mptcp_sock *msk = mptcp_sk(sk); int err = -EINVAL; struct sock *ssk; ssk = __mptcp_nmpc_sk(msk); if (IS_ERR(ssk)) return PTR_ERR(ssk); mptcp_set_state(sk, TCP_SYN_SENT); subflow = mptcp_subflow_ctx(ssk); #ifdef CONFIG_TCP_MD5SIG /* no MPTCP if MD5SIG is enabled on this socket or we may run out of * TCP option space. */ if (rcu_access_pointer(tcp_sk(ssk)->md5sig_info)) mptcp_early_fallback(msk, subflow, MPTCP_MIB_MD5SIGFALLBACK); #endif if (subflow->request_mptcp) { if (mptcp_active_should_disable(sk)) mptcp_early_fallback(msk, subflow, MPTCP_MIB_MPCAPABLEACTIVEDISABLED); else if (mptcp_token_new_connect(ssk) < 0) mptcp_early_fallback(msk, subflow, MPTCP_MIB_TOKENFALLBACKINIT); } WRITE_ONCE(msk->write_seq, subflow->idsn); WRITE_ONCE(msk->snd_nxt, subflow->idsn); WRITE_ONCE(msk->snd_una, subflow->idsn); if (likely(!__mptcp_check_fallback(msk))) MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVE); /* if reaching here via the fastopen/sendmsg path, the caller already * acquired the subflow socket lock, too. */ if (!msk->fastopening) lock_sock(ssk); /* the following mirrors closely a very small chunk of code from * __inet_stream_connect() */ if (ssk->sk_state != TCP_CLOSE) goto out; if (BPF_CGROUP_PRE_CONNECT_ENABLED(ssk)) { err = ssk->sk_prot->pre_connect(ssk, uaddr, addr_len); if (err) goto out; } err = ssk->sk_prot->connect(ssk, uaddr, addr_len); if (err < 0) goto out; inet_assign_bit(DEFER_CONNECT, sk, inet_test_bit(DEFER_CONNECT, ssk)); out: if (!msk->fastopening) release_sock(ssk); /* on successful connect, the msk state will be moved to established by * subflow_finish_connect() */ if (unlikely(err)) { /* avoid leaving a dangling token in an unconnected socket */ mptcp_token_destroy(msk); mptcp_set_state(sk, TCP_CLOSE); return err; } mptcp_copy_inaddrs(sk, ssk); return 0; } static struct proto mptcp_prot = { .name = "MPTCP", .owner = THIS_MODULE, .init = mptcp_init_sock, .connect = mptcp_connect, .disconnect = mptcp_disconnect, .close = mptcp_close, .setsockopt = mptcp_setsockopt, .getsockopt = mptcp_getsockopt, .shutdown = mptcp_shutdown, .destroy = mptcp_destroy, .sendmsg = mptcp_sendmsg, .ioctl = mptcp_ioctl, .recvmsg = mptcp_recvmsg, .release_cb = mptcp_release_cb, .hash = mptcp_hash, .unhash = mptcp_unhash, .get_port = mptcp_get_port, .stream_memory_free = mptcp_stream_memory_free, .sockets_allocated = &mptcp_sockets_allocated, .memory_allocated = &net_aligned_data.tcp_memory_allocated, .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, .memory_pressure = &tcp_memory_pressure, .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), .sysctl_mem = sysctl_tcp_mem, .obj_size = sizeof(struct mptcp_sock), .slab_flags = SLAB_TYPESAFE_BY_RCU, .no_autobind = true, }; static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) { struct mptcp_sock *msk = mptcp_sk(sock->sk); struct sock *ssk, *sk = sock->sk; int err = -EINVAL; lock_sock(sk); ssk = __mptcp_nmpc_sk(msk); if (IS_ERR(ssk)) { err = PTR_ERR(ssk); goto unlock; } if (sk->sk_family == AF_INET) err = inet_bind_sk(ssk, uaddr, addr_len); #if IS_ENABLED(CONFIG_MPTCP_IPV6) else if (sk->sk_family == AF_INET6) err = inet6_bind_sk(ssk, uaddr, addr_len); #endif if (!err) mptcp_copy_inaddrs(sk, ssk); unlock: release_sock(sk); return err; } static int mptcp_listen(struct socket *sock, int backlog) { struct mptcp_sock *msk = mptcp_sk(sock->sk); struct sock *sk = sock->sk; struct sock *ssk; int err; pr_debug("msk=%p\n", msk); lock_sock(sk); err = -EINVAL; if (sock->state != SS_UNCONNECTED || sock->type != SOCK_STREAM) goto unlock; ssk = __mptcp_nmpc_sk(msk); if (IS_ERR(ssk)) { err = PTR_ERR(ssk); goto unlock; } mptcp_set_state(sk, TCP_LISTEN); sock_set_flag(sk, SOCK_RCU_FREE); lock_sock(ssk); err = __inet_listen_sk(ssk, backlog); release_sock(ssk); mptcp_set_state(sk, inet_sk_state_load(ssk)); if (!err) { sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); mptcp_copy_inaddrs(sk, ssk); mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CREATED); } unlock: release_sock(sk); return err; } static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, struct proto_accept_arg *arg) { struct mptcp_sock *msk = mptcp_sk(sock->sk); struct sock *ssk, *newsk; pr_debug("msk=%p\n", msk); /* Buggy applications can call accept on socket states other then LISTEN * but no need to allocate the first subflow just to error out. */ ssk = READ_ONCE(msk->first); if (!ssk) return -EINVAL; pr_debug("ssk=%p, listener=%p\n", ssk, mptcp_subflow_ctx(ssk)); newsk = inet_csk_accept(ssk, arg); if (!newsk) return arg->err; pr_debug("newsk=%p, subflow is mptcp=%d\n", newsk, sk_is_mptcp(newsk)); if (sk_is_mptcp(newsk)) { struct mptcp_subflow_context *subflow; struct sock *new_mptcp_sock; subflow = mptcp_subflow_ctx(newsk); new_mptcp_sock = subflow->conn; /* is_mptcp should be false if subflow->conn is missing, see * subflow_syn_recv_sock() */ if (WARN_ON_ONCE(!new_mptcp_sock)) { tcp_sk(newsk)->is_mptcp = 0; goto tcpfallback; } newsk = new_mptcp_sock; MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPCAPABLEPASSIVEACK); newsk->sk_kern_sock = arg->kern; lock_sock(newsk); __inet_accept(sock, newsock, newsk); set_bit(SOCK_CUSTOM_SOCKOPT, &newsock->flags); msk = mptcp_sk(newsk); msk->in_accept_queue = 0; /* set ssk->sk_socket of accept()ed flows to mptcp socket. * This is needed so NOSPACE flag can be set from tcp stack. */ mptcp_for_each_subflow(msk, subflow) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); if (!ssk->sk_socket) mptcp_sock_graft(ssk, newsock); } mptcp_rps_record_subflows(msk); /* Do late cleanup for the first subflow as necessary. Also * deal with bad peers not doing a complete shutdown. */ if (unlikely(inet_sk_state_load(msk->first) == TCP_CLOSE)) { __mptcp_close_ssk(newsk, msk->first, mptcp_subflow_ctx(msk->first), 0); if (unlikely(list_is_singular(&msk->conn_list))) mptcp_set_state(newsk, TCP_CLOSE); } } else { tcpfallback: newsk->sk_kern_sock = arg->kern; lock_sock(newsk); __inet_accept(sock, newsock, newsk); /* we are being invoked after accepting a non-mp-capable * flow: sk is a tcp_sk, not an mptcp one. * * Hand the socket over to tcp so all further socket ops * bypass mptcp. */ WRITE_ONCE(newsock->sk->sk_socket->ops, mptcp_fallback_tcp_ops(newsock->sk)); } release_sock(newsk); return 0; } static __poll_t mptcp_check_writeable(struct mptcp_sock *msk) { struct sock *sk = (struct sock *)msk; if (__mptcp_stream_is_writeable(sk, 1)) return EPOLLOUT | EPOLLWRNORM; set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); smp_mb__after_atomic(); /* NOSPACE is changed by mptcp_write_space() */ if (__mptcp_stream_is_writeable(sk, 1)) return EPOLLOUT | EPOLLWRNORM; return 0; } static __poll_t mptcp_poll(struct file *file, struct socket *sock, struct poll_table_struct *wait) { struct sock *sk = sock->sk; struct mptcp_sock *msk; __poll_t mask = 0; u8 shutdown; int state; msk = mptcp_sk(sk); sock_poll_wait(file, sock, wait); state = inet_sk_state_load(sk); pr_debug("msk=%p state=%d flags=%lx\n", msk, state, msk->flags); if (state == TCP_LISTEN) { struct sock *ssk = READ_ONCE(msk->first); if (WARN_ON_ONCE(!ssk)) return 0; return inet_csk_listen_poll(ssk); } shutdown = READ_ONCE(sk->sk_shutdown); if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) mask |= EPOLLHUP; if (shutdown & RCV_SHUTDOWN) mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) { mask |= mptcp_check_readable(sk); if (shutdown & SEND_SHUTDOWN) mask |= EPOLLOUT | EPOLLWRNORM; else mask |= mptcp_check_writeable(msk); } else if (state == TCP_SYN_SENT && inet_test_bit(DEFER_CONNECT, sk)) { /* cf tcp_poll() note about TFO */ mask |= EPOLLOUT | EPOLLWRNORM; } /* This barrier is coupled with smp_wmb() in __mptcp_error_report() */ smp_rmb(); if (READ_ONCE(sk->sk_err)) mask |= EPOLLERR; return mask; } static const struct proto_ops mptcp_stream_ops = { .family = PF_INET, .owner = THIS_MODULE, .release = inet_release, .bind = mptcp_bind, .connect = inet_stream_connect, .socketpair = sock_no_socketpair, .accept = mptcp_stream_accept, .getname = inet_getname, .poll = mptcp_poll, .ioctl = inet_ioctl, .gettstamp = sock_gettstamp, .listen = mptcp_listen, .shutdown = inet_shutdown, .setsockopt = sock_common_setsockopt, .getsockopt = sock_common_getsockopt, .sendmsg = inet_sendmsg, .recvmsg = inet_recvmsg, .mmap = sock_no_mmap, .set_rcvlowat = mptcp_set_rcvlowat, }; static struct inet_protosw mptcp_protosw = { .type = SOCK_STREAM, .protocol = IPPROTO_MPTCP, .prot = &mptcp_prot, .ops = &mptcp_stream_ops, .flags = INET_PROTOSW_ICSK, }; static int mptcp_napi_poll(struct napi_struct *napi, int budget) { struct mptcp_delegated_action *delegated; struct mptcp_subflow_context *subflow; int work_done = 0; delegated = container_of(napi, struct mptcp_delegated_action, napi); while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) { struct sock *ssk = mptcp_subflow_tcp_sock(subflow); bh_lock_sock_nested(ssk); if (!sock_owned_by_user(ssk)) { mptcp_subflow_process_delegated(ssk, xchg(&subflow->delegated_status, 0)); } else { /* tcp_release_cb_override already processed * the action or will do at next release_sock(). * In both case must dequeue the subflow here - on the same * CPU that scheduled it. */ smp_wmb(); clear_bit(MPTCP_DELEGATE_SCHEDULED, &subflow->delegated_status); } bh_unlock_sock(ssk); sock_put(ssk); if (++work_done == budget) return budget; } /* always provide a 0 'work_done' argument, so that napi_complete_done * will not try accessing the NULL napi->dev ptr */ napi_complete_done(napi, 0); return work_done; } void __init mptcp_proto_init(void) { struct mptcp_delegated_action *delegated; int cpu; mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL)) panic("Failed to allocate MPTCP pcpu counter\n"); mptcp_napi_dev = alloc_netdev_dummy(0); if (!mptcp_napi_dev) panic("Failed to allocate MPTCP dummy netdev\n"); for_each_possible_cpu(cpu) { delegated = per_cpu_ptr(&mptcp_delegated_actions, cpu); INIT_LIST_HEAD(&delegated->head); netif_napi_add_tx(mptcp_napi_dev, &delegated->napi, mptcp_napi_poll); napi_enable(&delegated->napi); } mptcp_subflow_init(); mptcp_pm_init(); mptcp_sched_init(); mptcp_token_init(); if (proto_register(&mptcp_prot, 1) != 0) panic("Failed to register MPTCP proto.\n"); inet_register_protosw(&mptcp_protosw); BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); } #if IS_ENABLED(CONFIG_MPTCP_IPV6) static const struct proto_ops mptcp_v6_stream_ops = { .family = PF_INET6, .owner = THIS_MODULE, .release = inet6_release, .bind = mptcp_bind, .connect = inet_stream_connect, .socketpair = sock_no_socketpair, .accept = mptcp_stream_accept, .getname = inet6_getname, .poll = mptcp_poll, .ioctl = inet6_ioctl, .gettstamp = sock_gettstamp, .listen = mptcp_listen, .shutdown = inet_shutdown, .setsockopt = sock_common_setsockopt, .getsockopt = sock_common_getsockopt, .sendmsg = inet6_sendmsg, .recvmsg = inet6_recvmsg, .mmap = sock_no_mmap, #ifdef CONFIG_COMPAT .compat_ioctl = inet6_compat_ioctl, #endif .set_rcvlowat = mptcp_set_rcvlowat, }; static struct proto mptcp_v6_prot; static struct inet_protosw mptcp_v6_protosw = { .type = SOCK_STREAM, .protocol = IPPROTO_MPTCP, .prot = &mptcp_v6_prot, .ops = &mptcp_v6_stream_ops, .flags = INET_PROTOSW_ICSK, }; int __init mptcp_proto_v6_init(void) { int err; mptcp_v6_prot = mptcp_prot; strscpy(mptcp_v6_prot.name, "MPTCPv6", sizeof(mptcp_v6_prot.name)); mptcp_v6_prot.slab = NULL; mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); mptcp_v6_prot.ipv6_pinfo_offset = offsetof(struct mptcp6_sock, np); err = proto_register(&mptcp_v6_prot, 1); if (err) return err; err = inet6_register_protosw(&mptcp_v6_protosw); if (err) proto_unregister(&mptcp_v6_prot); return err; } #endif |
| 15 2 1 109 111 7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef __DRM_GEM_SHMEM_HELPER_H__ #define __DRM_GEM_SHMEM_HELPER_H__ #include <linux/fs.h> #include <linux/mm.h> #include <linux/mutex.h> #include <drm/drm_file.h> #include <drm/drm_gem.h> #include <drm/drm_ioctl.h> #include <drm/drm_prime.h> struct dma_buf_attachment; struct drm_mode_create_dumb; struct drm_printer; struct sg_table; /** * struct drm_gem_shmem_object - GEM object backed by shmem */ struct drm_gem_shmem_object { /** * @base: Base GEM object */ struct drm_gem_object base; /** * @pages: Page table */ struct page **pages; /** * @pages_use_count: * * Reference count on the pages table. * The pages are put when the count reaches zero. */ refcount_t pages_use_count; /** * @pages_pin_count: * * Reference count on the pinned pages table. * * Pages are hard-pinned and reside in memory if count * greater than zero. Otherwise, when count is zero, the pages are * allowed to be evicted and purged by memory shrinker. */ refcount_t pages_pin_count; /** * @madv: State for madvise * * 0 is active/inuse. * A negative value is the object is purged. * Positive values are driver specific and not used by the helpers. */ int madv; /** * @madv_list: List entry for madvise tracking * * Typically used by drivers to track purgeable objects */ struct list_head madv_list; /** * @sgt: Scatter/gather table for imported PRIME buffers */ struct sg_table *sgt; /** * @vaddr: Kernel virtual address of the backing memory */ void *vaddr; /** * @vmap_use_count: * * Reference count on the virtual address. * The address are un-mapped when the count reaches zero. */ refcount_t vmap_use_count; /** * @pages_mark_dirty_on_put: * * Mark pages as dirty when they are put. */ bool pages_mark_dirty_on_put : 1; /** * @pages_mark_accessed_on_put: * * Mark pages as accessed when they are put. */ bool pages_mark_accessed_on_put : 1; /** * @map_wc: map object write-combined (instead of using shmem defaults). */ bool map_wc : 1; }; #define to_drm_gem_shmem_obj(obj) \ container_of(obj, struct drm_gem_shmem_object, base) struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size); struct drm_gem_shmem_object *drm_gem_shmem_create_with_mnt(struct drm_device *dev, size_t size, struct vfsmount *gemfs); void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem); void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem); int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem); void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem); int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, struct iosys_map *map); void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, struct iosys_map *map); int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct *vma); int drm_gem_shmem_pin_locked(struct drm_gem_shmem_object *shmem); void drm_gem_shmem_unpin_locked(struct drm_gem_shmem_object *shmem); int drm_gem_shmem_madvise_locked(struct drm_gem_shmem_object *shmem, int madv); static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem) { return (shmem->madv > 0) && !refcount_read(&shmem->pages_pin_count) && shmem->sgt && !shmem->base.dma_buf && !drm_gem_is_imported(&shmem->base); } void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem); struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem); struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem); void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem, struct drm_printer *p, unsigned int indent); extern const struct vm_operations_struct drm_gem_shmem_vm_ops; /* * GEM object functions */ /** * drm_gem_shmem_object_free - GEM object function for drm_gem_shmem_free() * @obj: GEM object to free * * This function wraps drm_gem_shmem_free(). Drivers that employ the shmem helpers * should use it as their &drm_gem_object_funcs.free handler. */ static inline void drm_gem_shmem_object_free(struct drm_gem_object *obj) { struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); drm_gem_shmem_free(shmem); } /** * drm_gem_shmem_object_print_info() - Print &drm_gem_shmem_object info for debugfs * @p: DRM printer * @indent: Tab indentation level * @obj: GEM object * * This function wraps drm_gem_shmem_print_info(). Drivers that employ the shmem helpers should * use this function as their &drm_gem_object_funcs.print_info handler. */ static inline void drm_gem_shmem_object_print_info(struct drm_printer *p, unsigned int indent, const struct drm_gem_object *obj) { const struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); drm_gem_shmem_print_info(shmem, p, indent); } /** * drm_gem_shmem_object_pin - GEM object function for drm_gem_shmem_pin() * @obj: GEM object * * This function wraps drm_gem_shmem_pin(). Drivers that employ the shmem helpers should * use it as their &drm_gem_object_funcs.pin handler. */ static inline int drm_gem_shmem_object_pin(struct drm_gem_object *obj) { struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); return drm_gem_shmem_pin_locked(shmem); } /** * drm_gem_shmem_object_unpin - GEM object function for drm_gem_shmem_unpin() * @obj: GEM object * * This function wraps drm_gem_shmem_unpin(). Drivers that employ the shmem helpers should * use it as their &drm_gem_object_funcs.unpin handler. */ static inline void drm_gem_shmem_object_unpin(struct drm_gem_object *obj) { struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); drm_gem_shmem_unpin_locked(shmem); } /** * drm_gem_shmem_object_get_sg_table - GEM object function for drm_gem_shmem_get_sg_table() * @obj: GEM object * * This function wraps drm_gem_shmem_get_sg_table(). Drivers that employ the shmem helpers should * use it as their &drm_gem_object_funcs.get_sg_table handler. * * Returns: * A pointer to the scatter/gather table of pinned pages or error pointer on failure. */ static inline struct sg_table *drm_gem_shmem_object_get_sg_table(struct drm_gem_object *obj) { struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); return drm_gem_shmem_get_sg_table(shmem); } /* * drm_gem_shmem_object_vmap - GEM object function for drm_gem_shmem_vmap_locked() * @obj: GEM object * @map: Returns the kernel virtual address of the SHMEM GEM object's backing store. * * This function wraps drm_gem_shmem_vmap_locked(). Drivers that employ the shmem * helpers should use it as their &drm_gem_object_funcs.vmap handler. * * Returns: * 0 on success or a negative error code on failure. */ static inline int drm_gem_shmem_object_vmap(struct drm_gem_object *obj, struct iosys_map *map) { struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); return drm_gem_shmem_vmap_locked(shmem, map); } /* * drm_gem_shmem_object_vunmap - GEM object function for drm_gem_shmem_vunmap() * @obj: GEM object * @map: Kernel virtual address where the SHMEM GEM object was mapped * * This function wraps drm_gem_shmem_vunmap_locked(). Drivers that employ the shmem * helpers should use it as their &drm_gem_object_funcs.vunmap handler. */ static inline void drm_gem_shmem_object_vunmap(struct drm_gem_object *obj, struct iosys_map *map) { struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); drm_gem_shmem_vunmap_locked(shmem, map); } /** * drm_gem_shmem_object_mmap - GEM object function for drm_gem_shmem_mmap() * @obj: GEM object * @vma: VMA for the area to be mapped * * This function wraps drm_gem_shmem_mmap(). Drivers that employ the shmem helpers should * use it as their &drm_gem_object_funcs.mmap handler. * * Returns: * 0 on success or a negative error code on failure. */ static inline int drm_gem_shmem_object_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) { struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); return drm_gem_shmem_mmap(shmem, vma); } /* * Driver ops */ struct drm_gem_object * drm_gem_shmem_prime_import_sg_table(struct drm_device *dev, struct dma_buf_attachment *attach, struct sg_table *sgt); int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev, struct drm_mode_create_dumb *args); struct drm_gem_object *drm_gem_shmem_prime_import_no_map(struct drm_device *dev, struct dma_buf *buf); /** * DRM_GEM_SHMEM_DRIVER_OPS - Default shmem GEM operations * * This macro provides a shortcut for setting the shmem GEM operations * in the &drm_driver structure. Drivers that do not require an s/g table * for imported buffers should use this. */ #define DRM_GEM_SHMEM_DRIVER_OPS \ .gem_prime_import = drm_gem_shmem_prime_import_no_map, \ .dumb_create = drm_gem_shmem_dumb_create #endif /* __DRM_GEM_SHMEM_HELPER_H__ */ |
| 198 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _LINUX_VIRTIO_RING_H #define _LINUX_VIRTIO_RING_H #include <asm/barrier.h> #include <linux/virtio.h> #include <linux/irqreturn.h> #include <uapi/linux/virtio_ring.h> /* * Barriers in virtio are tricky. Non-SMP virtio guests can't assume * they're not on an SMP host system, so they need to assume real * barriers. Non-SMP virtio hosts could skip the barriers, but does * anyone care? * * For virtio_pci on SMP, we don't need to order with respect to MMIO * accesses through relaxed memory I/O windows, so virt_mb() et al are * sufficient. * * For using virtio to talk to real devices (eg. other heterogeneous * CPUs) we do need real barriers. In theory, we could be using both * kinds of virtio, so it's a runtime decision, and the branch is * actually quite cheap. */ static inline void virtio_mb(bool weak_barriers) { if (weak_barriers) virt_mb(); else mb(); } static inline void virtio_rmb(bool weak_barriers) { if (weak_barriers) virt_rmb(); else dma_rmb(); } static inline void virtio_wmb(bool weak_barriers) { if (weak_barriers) virt_wmb(); else dma_wmb(); } #define virtio_store_mb(weak_barriers, p, v) \ do { \ if (weak_barriers) { \ virt_store_mb(*p, v); \ } else { \ WRITE_ONCE(*p, v); \ mb(); \ } \ } while (0) \ struct virtio_device; struct virtqueue; struct device; /* * Creates a virtqueue and allocates the descriptor ring. If * may_reduce_num is set, then this may allocate a smaller ring than * expected. The caller should query virtqueue_get_vring_size to learn * the actual size of the ring. */ struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, unsigned int vring_align, struct virtio_device *vdev, bool weak_barriers, bool may_reduce_num, bool ctx, bool (*notify)(struct virtqueue *vq), void (*callback)(struct virtqueue *vq), const char *name); /* * Creates a virtqueue and allocates the descriptor ring with per * virtqueue mapping operations. */ struct virtqueue *vring_create_virtqueue_map(unsigned int index, unsigned int num, unsigned int vring_align, struct virtio_device *vdev, bool weak_barriers, bool may_reduce_num, bool ctx, bool (*notify)(struct virtqueue *vq), void (*callback)(struct virtqueue *vq), const char *name, union virtio_map map); /* * Creates a virtqueue with a standard layout but a caller-allocated * ring. */ struct virtqueue *vring_new_virtqueue(unsigned int index, unsigned int num, unsigned int vring_align, struct virtio_device *vdev, bool weak_barriers, bool ctx, void *pages, bool (*notify)(struct virtqueue *vq), void (*callback)(struct virtqueue *vq), const char *name); /* * Destroys a virtqueue. If created with vring_create_virtqueue, this * also frees the ring. */ void vring_del_virtqueue(struct virtqueue *vq); /* Filter out transport-specific feature bits. */ void vring_transport_features(struct virtio_device *vdev); irqreturn_t vring_interrupt(int irq, void *_vq); u32 vring_notification_data(struct virtqueue *_vq); #endif /* _LINUX_VIRTIO_RING_H */ |
| 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | /* SPDX-License-Identifier: GPL-2.0-only */ /* * Landlock - Filesystem management and hooks * * Copyright © 2017-2020 Mickaël Salaün <mic@digikod.net> * Copyright © 2018-2020 ANSSI * Copyright © 2024-2025 Microsoft Corporation */ #ifndef _SECURITY_LANDLOCK_FS_H #define _SECURITY_LANDLOCK_FS_H #include <linux/build_bug.h> #include <linux/fs.h> #include <linux/init.h> #include <linux/rcupdate.h> #include "access.h" #include "cred.h" #include "ruleset.h" #include "setup.h" /** * struct landlock_inode_security - Inode security blob * * Enable to reference a &struct landlock_object tied to an inode (i.e. * underlying object). */ struct landlock_inode_security { /** * @object: Weak pointer to an allocated object. All assignments of a * new object are protected by the underlying inode->i_lock. However, * atomically disassociating @object from the inode is only protected * by @object->lock, from the time @object's usage refcount drops to * zero to the time this pointer is nulled out (cf. release_inode() and * hook_sb_delete()). Indeed, such disassociation doesn't require * inode->i_lock thanks to the careful rcu_access_pointer() check * performed by get_inode_object(). */ struct landlock_object __rcu *object; }; /** * struct landlock_file_security - File security blob * * This information is populated when opening a file in hook_file_open, and * tracks the relevant Landlock access rights that were available at the time * of opening the file. Other LSM hooks use these rights in order to authorize * operations on already opened files. */ struct landlock_file_security { /** * @allowed_access: Access rights that were available at the time of * opening the file. This is not necessarily the full set of access * rights available at that time, but it's the necessary subset as * needed to authorize later operations on the open file. */ access_mask_t allowed_access; #ifdef CONFIG_AUDIT /** * @deny_masks: Domain layer levels that deny an optional access (see * _LANDLOCK_ACCESS_FS_OPTIONAL). */ deny_masks_t deny_masks; /** * @fown_layer: Layer level of @fown_subject->domain with * LANDLOCK_SCOPE_SIGNAL. */ u8 fown_layer; #endif /* CONFIG_AUDIT */ /** * @fown_subject: Landlock credential of the task that set the PID that * may receive a signal e.g., SIGURG when writing MSG_OOB to the * related socket. This pointer is protected by the related * file->f_owner->lock, as for fown_struct's members: pid, uid, and * euid. */ struct landlock_cred_security fown_subject; }; #ifdef CONFIG_AUDIT /* Makes sure all layers can be identified. */ /* clang-format off */ static_assert((typeof_member(struct landlock_file_security, fown_layer))~0 >= LANDLOCK_MAX_NUM_LAYERS); /* clang-format off */ #endif /* CONFIG_AUDIT */ /** * struct landlock_superblock_security - Superblock security blob * * Enable hook_sb_delete() to wait for concurrent calls to release_inode(). */ struct landlock_superblock_security { /** * @inode_refs: Number of pending inodes (from this superblock) that * are being released by release_inode(). * Cf. struct super_block->s_fsnotify_inode_refs . */ atomic_long_t inode_refs; }; static inline struct landlock_file_security * landlock_file(const struct file *const file) { return file->f_security + landlock_blob_sizes.lbs_file; } static inline struct landlock_inode_security * landlock_inode(const struct inode *const inode) { return inode->i_security + landlock_blob_sizes.lbs_inode; } static inline struct landlock_superblock_security * landlock_superblock(const struct super_block *const superblock) { return superblock->s_security + landlock_blob_sizes.lbs_superblock; } __init void landlock_add_fs_hooks(void); int landlock_append_fs_rule(struct landlock_ruleset *const ruleset, const struct path *const path, access_mask_t access_hierarchy); #endif /* _SECURITY_LANDLOCK_FS_H */ |
| 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Stadia controller rumble support. * * Copyright 2023 Google LLC */ #include <linux/hid.h> #include <linux/input.h> #include <linux/slab.h> #include <linux/module.h> #include "hid-ids.h" #define STADIA_FF_REPORT_ID 5 struct stadiaff_device { struct hid_device *hid; struct hid_report *report; spinlock_t lock; bool removed; uint16_t strong_magnitude; uint16_t weak_magnitude; struct work_struct work; }; static void stadiaff_work(struct work_struct *work) { struct stadiaff_device *stadiaff = container_of(work, struct stadiaff_device, work); struct hid_field *rumble_field = stadiaff->report->field[0]; unsigned long flags; spin_lock_irqsave(&stadiaff->lock, flags); rumble_field->value[0] = stadiaff->strong_magnitude; rumble_field->value[1] = stadiaff->weak_magnitude; spin_unlock_irqrestore(&stadiaff->lock, flags); hid_hw_request(stadiaff->hid, stadiaff->report, HID_REQ_SET_REPORT); } static int stadiaff_play(struct input_dev *dev, void *data, struct ff_effect *effect) { struct hid_device *hid = input_get_drvdata(dev); struct stadiaff_device *stadiaff = hid_get_drvdata(hid); unsigned long flags; spin_lock_irqsave(&stadiaff->lock, flags); if (!stadiaff->removed) { stadiaff->strong_magnitude = effect->u.rumble.strong_magnitude; stadiaff->weak_magnitude = effect->u.rumble.weak_magnitude; schedule_work(&stadiaff->work); } spin_unlock_irqrestore(&stadiaff->lock, flags); return 0; } static int stadiaff_init(struct hid_device *hid) { struct stadiaff_device *stadiaff; struct hid_report *report; struct hid_input *hidinput; struct input_dev *dev; int error; if (list_empty(&hid->inputs)) { hid_err(hid, "no inputs found\n"); return -ENODEV; } hidinput = list_entry(hid->inputs.next, struct hid_input, list); dev = hidinput->input; report = hid_validate_values(hid, HID_OUTPUT_REPORT, STADIA_FF_REPORT_ID, 0, 2); if (!report) return -ENODEV; stadiaff = devm_kzalloc(&hid->dev, sizeof(struct stadiaff_device), GFP_KERNEL); if (!stadiaff) return -ENOMEM; hid_set_drvdata(hid, stadiaff); input_set_capability(dev, EV_FF, FF_RUMBLE); error = input_ff_create_memless(dev, NULL, stadiaff_play); if (error) return error; stadiaff->removed = false; stadiaff->hid = hid; stadiaff->report = report; INIT_WORK(&stadiaff->work, stadiaff_work); spin_lock_init(&stadiaff->lock); hid_info(hid, "Force Feedback for Google Stadia controller\n"); return 0; } static int stadia_probe(struct hid_device *hdev, const struct hid_device_id *id) { int ret; ret = hid_parse(hdev); if (ret) { hid_err(hdev, "parse failed\n"); return ret; } ret = hid_hw_start(hdev, HID_CONNECT_DEFAULT & ~HID_CONNECT_FF); if (ret) { hid_err(hdev, "hw start failed\n"); return ret; } ret = stadiaff_init(hdev); if (ret) { hid_err(hdev, "force feedback init failed\n"); hid_hw_stop(hdev); return ret; } return 0; } static void stadia_remove(struct hid_device *hid) { struct stadiaff_device *stadiaff = hid_get_drvdata(hid); unsigned long flags; spin_lock_irqsave(&stadiaff->lock, flags); stadiaff->removed = true; spin_unlock_irqrestore(&stadiaff->lock, flags); cancel_work_sync(&stadiaff->work); hid_hw_stop(hid); } static const struct hid_device_id stadia_devices[] = { { HID_USB_DEVICE(USB_VENDOR_ID_GOOGLE, USB_DEVICE_ID_GOOGLE_STADIA) }, { HID_BLUETOOTH_DEVICE(USB_VENDOR_ID_GOOGLE, USB_DEVICE_ID_GOOGLE_STADIA) }, { } }; MODULE_DEVICE_TABLE(hid, stadia_devices); static struct hid_driver stadia_driver = { .name = "stadia", .id_table = stadia_devices, .probe = stadia_probe, .remove = stadia_remove, }; module_hid_driver(stadia_driver); MODULE_DESCRIPTION("Google Stadia controller rumble support."); MODULE_LICENSE("GPL"); |
| 2 2 2 2 2 2 2 2 2 21 1 1 1 2 2 2 2 26 26 2 24 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Clock domain and sample rate management functions */ #include <linux/bitops.h> #include <linux/init.h> #include <linux/string.h> #include <linux/usb.h> #include <linux/usb/audio.h> #include <linux/usb/audio-v2.h> #include <linux/usb/audio-v3.h> #include <sound/core.h> #include <sound/info.h> #include <sound/pcm.h> #include "usbaudio.h" #include "card.h" #include "helper.h" #include "clock.h" #include "quirks.h" union uac23_clock_source_desc { struct uac_clock_source_descriptor v2; struct uac3_clock_source_descriptor v3; }; union uac23_clock_selector_desc { struct uac_clock_selector_descriptor v2; struct uac3_clock_selector_descriptor v3; }; union uac23_clock_multiplier_desc { struct uac_clock_multiplier_descriptor v2; struct uac_clock_multiplier_descriptor v3; }; /* check whether the descriptor bLength has the minimal length */ #define DESC_LENGTH_CHECK(p, proto) \ ((proto) == UAC_VERSION_3 ? \ ((p)->v3.bLength >= sizeof((p)->v3)) : \ ((p)->v2.bLength >= sizeof((p)->v2))) #define GET_VAL(p, proto, field) \ ((proto) == UAC_VERSION_3 ? (p)->v3.field : (p)->v2.field) static void *find_uac_clock_desc(struct usb_host_interface *iface, int id, bool (*validator)(void *, int, int), u8 type, int proto) { void *cs = NULL; while ((cs = snd_usb_find_csint_desc(iface->extra, iface->extralen, cs, type))) { if (validator(cs, id, proto)) return cs; } return NULL; } static bool validate_clock_source(void *p, int id, int proto) { union uac23_clock_source_desc *cs = p; if (!DESC_LENGTH_CHECK(cs, proto)) return false; return GET_VAL(cs, proto, bClockID) == id; } static bool validate_clock_selector(void *p, int id, int proto) { union uac23_clock_selector_desc *cs = p; if (!DESC_LENGTH_CHECK(cs, proto)) return false; if (GET_VAL(cs, proto, bClockID) != id) return false; /* additional length check for baCSourceID array (in bNrInPins size) * and two more fields (which sizes depend on the protocol) */ if (proto == UAC_VERSION_3) return cs->v3.bLength >= sizeof(cs->v3) + cs->v3.bNrInPins + 4 /* bmControls */ + 2 /* wCSelectorDescrStr */; else return cs->v2.bLength >= sizeof(cs->v2) + cs->v2.bNrInPins + 1 /* bmControls */ + 1 /* iClockSelector */; } static bool validate_clock_multiplier(void *p, int id, int proto) { union uac23_clock_multiplier_desc *cs = p; if (!DESC_LENGTH_CHECK(cs, proto)) return false; return GET_VAL(cs, proto, bClockID) == id; } #define DEFINE_FIND_HELPER(name, obj, validator, type2, type3) \ static obj *name(struct snd_usb_audio *chip, int id, \ const struct audioformat *fmt) \ { \ struct usb_host_interface *ctrl_intf = \ snd_usb_find_ctrl_interface(chip, fmt->iface); \ return find_uac_clock_desc(ctrl_intf, id, validator, \ fmt->protocol == UAC_VERSION_3 ? (type3) : (type2), \ fmt->protocol); \ } DEFINE_FIND_HELPER(snd_usb_find_clock_source, union uac23_clock_source_desc, validate_clock_source, UAC2_CLOCK_SOURCE, UAC3_CLOCK_SOURCE); DEFINE_FIND_HELPER(snd_usb_find_clock_selector, union uac23_clock_selector_desc, validate_clock_selector, UAC2_CLOCK_SELECTOR, UAC3_CLOCK_SELECTOR); DEFINE_FIND_HELPER(snd_usb_find_clock_multiplier, union uac23_clock_multiplier_desc, validate_clock_multiplier, UAC2_CLOCK_MULTIPLIER, UAC3_CLOCK_MULTIPLIER); static int uac_clock_selector_get_val(struct snd_usb_audio *chip, int selector_id, int iface_no) { struct usb_host_interface *ctrl_intf; unsigned char buf; int ret; ctrl_intf = snd_usb_find_ctrl_interface(chip, iface_no); ret = snd_usb_ctl_msg(chip->dev, usb_rcvctrlpipe(chip->dev, 0), UAC2_CS_CUR, USB_RECIP_INTERFACE | USB_TYPE_CLASS | USB_DIR_IN, UAC2_CX_CLOCK_SELECTOR << 8, snd_usb_ctrl_intf(ctrl_intf) | (selector_id << 8), &buf, sizeof(buf)); if (ret < 0) return ret; return buf; } static int uac_clock_selector_set_val(struct snd_usb_audio *chip, int selector_id, unsigned char pin, int iface_no) { struct usb_host_interface *ctrl_intf; int ret; ctrl_intf = snd_usb_find_ctrl_interface(chip, iface_no); ret = snd_usb_ctl_msg(chip->dev, usb_sndctrlpipe(chip->dev, 0), UAC2_CS_CUR, USB_RECIP_INTERFACE | USB_TYPE_CLASS | USB_DIR_OUT, UAC2_CX_CLOCK_SELECTOR << 8, snd_usb_ctrl_intf(ctrl_intf) | (selector_id << 8), &pin, sizeof(pin)); if (ret < 0) return ret; if (ret != sizeof(pin)) { usb_audio_err(chip, "setting selector (id %d) unexpected length %d\n", selector_id, ret); return -EINVAL; } ret = uac_clock_selector_get_val(chip, selector_id, iface_no); if (ret < 0) return ret; if (ret != pin) { usb_audio_err(chip, "setting selector (id %d) to %x failed (current: %d)\n", selector_id, pin, ret); return -EINVAL; } return ret; } static bool uac_clock_source_is_valid_quirk(struct snd_usb_audio *chip, const struct audioformat *fmt, int source_id) { bool ret = false; int count; unsigned char data; struct usb_device *dev = chip->dev; union uac23_clock_source_desc *cs_desc; struct usb_host_interface *ctrl_intf; ctrl_intf = snd_usb_find_ctrl_interface(chip, fmt->iface); cs_desc = snd_usb_find_clock_source(chip, source_id, fmt); if (!cs_desc) return false; if (fmt->protocol == UAC_VERSION_2) { /* * Assume the clock is valid if clock source supports only one * single sample rate, the terminal is connected directly to it * (there is no clock selector) and clock type is internal. * This is to deal with some Denon DJ controllers that always * reports that clock is invalid. */ if (fmt->nr_rates == 1 && (fmt->clock & 0xff) == cs_desc->v2.bClockID && (cs_desc->v2.bmAttributes & 0x3) != UAC_CLOCK_SOURCE_TYPE_EXT) return true; } /* * MOTU MicroBook IIc * Sample rate changes takes more than 2 seconds for this device. Clock * validity request returns false during that period. */ if (chip->usb_id == USB_ID(0x07fd, 0x0004)) { count = 0; while ((!ret) && (count < 50)) { int err; msleep(100); err = snd_usb_ctl_msg(dev, usb_rcvctrlpipe(dev, 0), UAC2_CS_CUR, USB_TYPE_CLASS | USB_RECIP_INTERFACE | USB_DIR_IN, UAC2_CS_CONTROL_CLOCK_VALID << 8, snd_usb_ctrl_intf(ctrl_intf) | (source_id << 8), &data, sizeof(data)); if (err < 0) { dev_warn(&dev->dev, "%s(): cannot get clock validity for id %d\n", __func__, source_id); return false; } ret = !!data; count++; } } return ret; } static bool uac_clock_source_is_valid(struct snd_usb_audio *chip, const struct audioformat *fmt, int source_id) { int err; unsigned char data; struct usb_device *dev = chip->dev; u32 bmControls; union uac23_clock_source_desc *cs_desc; struct usb_host_interface *ctrl_intf; ctrl_intf = snd_usb_find_ctrl_interface(chip, fmt->iface); cs_desc = snd_usb_find_clock_source(chip, source_id, fmt); if (!cs_desc) return false; if (fmt->protocol == UAC_VERSION_3) bmControls = le32_to_cpu(cs_desc->v3.bmControls); else bmControls = cs_desc->v2.bmControls; /* If a clock source can't tell us whether it's valid, we assume it is */ if (!uac_v2v3_control_is_readable(bmControls, UAC2_CS_CONTROL_CLOCK_VALID)) return true; err = snd_usb_ctl_msg(dev, usb_rcvctrlpipe(dev, 0), UAC2_CS_CUR, USB_TYPE_CLASS | USB_RECIP_INTERFACE | USB_DIR_IN, UAC2_CS_CONTROL_CLOCK_VALID << 8, snd_usb_ctrl_intf(ctrl_intf) | (source_id << 8), &data, sizeof(data)); if (err < 0) { dev_warn(&dev->dev, "%s(): cannot get clock validity for id %d\n", __func__, source_id); return false; } if (data) return true; else return uac_clock_source_is_valid_quirk(chip, fmt, source_id); } static int __uac_clock_find_source(struct snd_usb_audio *chip, const struct audioformat *fmt, int entity_id, unsigned long *visited, bool validate) { union uac23_clock_source_desc *source; union uac23_clock_selector_desc *selector; union uac23_clock_multiplier_desc *multiplier; int ret, i, cur, err, pins, clock_id; const u8 *sources; int proto = fmt->protocol; bool readable, writeable; u32 bmControls; entity_id &= 0xff; if (test_and_set_bit(entity_id, visited)) { usb_audio_warn(chip, "%s(): recursive clock topology detected, id %d.\n", __func__, entity_id); return -EINVAL; } /* first, see if the ID we're looking at is a clock source already */ source = snd_usb_find_clock_source(chip, entity_id, fmt); if (source) { entity_id = GET_VAL(source, proto, bClockID); if (validate && !uac_clock_source_is_valid(chip, fmt, entity_id)) { usb_audio_err(chip, "clock source %d is not valid, cannot use\n", entity_id); return -ENXIO; } return entity_id; } selector = snd_usb_find_clock_selector(chip, entity_id, fmt); if (selector) { pins = GET_VAL(selector, proto, bNrInPins); clock_id = GET_VAL(selector, proto, bClockID); sources = GET_VAL(selector, proto, baCSourceID); cur = 0; if (proto == UAC_VERSION_3) bmControls = le32_to_cpu(*(__le32 *)(&selector->v3.baCSourceID[0] + pins)); else bmControls = *(__u8 *)(&selector->v2.baCSourceID[0] + pins); readable = uac_v2v3_control_is_readable(bmControls, UAC2_CX_CLOCK_SELECTOR); writeable = uac_v2v3_control_is_writeable(bmControls, UAC2_CX_CLOCK_SELECTOR); if (pins == 1) { ret = 1; goto find_source; } /* for now just warn about buggy device */ if (!readable) usb_audio_warn(chip, "%s(): clock selector control is not readable, id %d\n", __func__, clock_id); /* the entity ID we are looking at is a selector. * find out what it currently selects */ ret = uac_clock_selector_get_val(chip, clock_id, fmt->iface); if (ret < 0) { if (!chip->autoclock) return ret; goto find_others; } /* Selector values are one-based */ if (ret > pins || ret < 1) { usb_audio_err(chip, "%s(): selector reported illegal value, id %d, ret %d\n", __func__, clock_id, ret); if (!chip->autoclock) return -EINVAL; goto find_others; } find_source: cur = ret; ret = __uac_clock_find_source(chip, fmt, sources[ret - 1], visited, validate); if (ret > 0) { /* Skip setting clock selector again for some devices */ if (chip->quirk_flags & QUIRK_FLAG_SKIP_CLOCK_SELECTOR || !writeable) return ret; err = uac_clock_selector_set_val(chip, entity_id, cur, fmt->iface); if (err < 0) { if (pins == 1) { usb_audio_dbg(chip, "%s(): selector returned an error, " "assuming a firmware bug, id %d, ret %d\n", __func__, clock_id, err); return ret; } return err; } } if (!validate || ret > 0 || !chip->autoclock) return ret; find_others: if (!writeable) return -ENXIO; /* The current clock source is invalid, try others. */ for (i = 1; i <= pins; i++) { if (i == cur) continue; ret = __uac_clock_find_source(chip, fmt, sources[i - 1], visited, true); if (ret < 0) continue; err = uac_clock_selector_set_val(chip, entity_id, i, fmt->iface); if (err < 0) continue; usb_audio_info(chip, "found and selected valid clock source %d\n", ret); return ret; } return -ENXIO; } /* FIXME: multipliers only act as pass-thru element for now */ multiplier = snd_usb_find_clock_multiplier(chip, entity_id, fmt); if (multiplier) return __uac_clock_find_source(chip, fmt, GET_VAL(multiplier, proto, bCSourceID), visited, validate); return -EINVAL; } /* * For all kinds of sample rate settings and other device queries, * the clock source (end-leaf) must be used. However, clock selectors, * clock multipliers and sample rate converters may be specified as * clock source input to terminal. This functions walks the clock path * to its end and tries to find the source. * * The 'visited' bitfield is used internally to detect recursive loops. * * Returns the clock source UnitID (>=0) on success, or an error. */ int snd_usb_clock_find_source(struct snd_usb_audio *chip, const struct audioformat *fmt, bool validate) { DECLARE_BITMAP(visited, 256); memset(visited, 0, sizeof(visited)); switch (fmt->protocol) { case UAC_VERSION_2: case UAC_VERSION_3: return __uac_clock_find_source(chip, fmt, fmt->clock, visited, validate); default: return -EINVAL; } } static int set_sample_rate_v1(struct snd_usb_audio *chip, const struct audioformat *fmt, int rate) { struct usb_device *dev = chip->dev; unsigned char data[3]; int err, crate; /* if endpoint doesn't have sampling rate control, bail out */ if (!(fmt->attributes & UAC_EP_CS_ATTR_SAMPLE_RATE)) return 0; data[0] = rate; data[1] = rate >> 8; data[2] = rate >> 16; err = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), UAC_SET_CUR, USB_TYPE_CLASS | USB_RECIP_ENDPOINT | USB_DIR_OUT, UAC_EP_CS_ATTR_SAMPLE_RATE << 8, fmt->endpoint, data, sizeof(data)); if (err < 0) { dev_err(&dev->dev, "%d:%d: cannot set freq %d to ep %#x\n", fmt->iface, fmt->altsetting, rate, fmt->endpoint); return err; } /* Don't check the sample rate for devices which we know don't * support reading */ if (chip->quirk_flags & QUIRK_FLAG_GET_SAMPLE_RATE) return 0; /* the firmware is likely buggy, don't repeat to fail too many times */ if (chip->sample_rate_read_error > 2) return 0; err = snd_usb_ctl_msg(dev, usb_rcvctrlpipe(dev, 0), UAC_GET_CUR, USB_TYPE_CLASS | USB_RECIP_ENDPOINT | USB_DIR_IN, UAC_EP_CS_ATTR_SAMPLE_RATE << 8, fmt->endpoint, data, sizeof(data)); if (err < 0) { dev_err(&dev->dev, "%d:%d: cannot get freq at ep %#x\n", fmt->iface, fmt->altsetting, fmt->endpoint); chip->sample_rate_read_error++; return 0; /* some devices don't support reading */ } crate = data[0] | (data[1] << 8) | (data[2] << 16); if (!crate) { dev_info(&dev->dev, "failed to read current rate; disabling the check\n"); chip->sample_rate_read_error = 3; /* three strikes, see above */ return 0; } if (crate != rate) { dev_warn(&dev->dev, "current rate %d is different from the runtime rate %d\n", crate, rate); // runtime->rate = crate; } return 0; } static int get_sample_rate_v2v3(struct snd_usb_audio *chip, int iface, int altsetting, int clock) { struct usb_device *dev = chip->dev; __le32 data; int err; struct usb_host_interface *ctrl_intf; ctrl_intf = snd_usb_find_ctrl_interface(chip, iface); err = snd_usb_ctl_msg(dev, usb_rcvctrlpipe(dev, 0), UAC2_CS_CUR, USB_TYPE_CLASS | USB_RECIP_INTERFACE | USB_DIR_IN, UAC2_CS_CONTROL_SAM_FREQ << 8, snd_usb_ctrl_intf(ctrl_intf) | (clock << 8), &data, sizeof(data)); if (err < 0) { dev_warn(&dev->dev, "%d:%d: cannot get freq (v2/v3): err %d\n", iface, altsetting, err); return 0; } return le32_to_cpu(data); } /* * Try to set the given sample rate: * * Return 0 if the clock source is read-only, the actual rate on success, * or a negative error code. * * This function gets called from format.c to validate each sample rate, too. * Hence no message is shown upon error */ int snd_usb_set_sample_rate_v2v3(struct snd_usb_audio *chip, const struct audioformat *fmt, int clock, int rate) { bool writeable; u32 bmControls; __le32 data; int err; union uac23_clock_source_desc *cs_desc; struct usb_host_interface *ctrl_intf; ctrl_intf = snd_usb_find_ctrl_interface(chip, fmt->iface); cs_desc = snd_usb_find_clock_source(chip, clock, fmt); if (!cs_desc) return 0; if (fmt->protocol == UAC_VERSION_3) bmControls = le32_to_cpu(cs_desc->v3.bmControls); else bmControls = cs_desc->v2.bmControls; writeable = uac_v2v3_control_is_writeable(bmControls, UAC2_CS_CONTROL_SAM_FREQ); if (!writeable) return 0; data = cpu_to_le32(rate); err = snd_usb_ctl_msg(chip->dev, usb_sndctrlpipe(chip->dev, 0), UAC2_CS_CUR, USB_TYPE_CLASS | USB_RECIP_INTERFACE | USB_DIR_OUT, UAC2_CS_CONTROL_SAM_FREQ << 8, snd_usb_ctrl_intf(ctrl_intf) | (clock << 8), &data, sizeof(data)); if (err < 0) return err; return get_sample_rate_v2v3(chip, fmt->iface, fmt->altsetting, clock); } static int set_sample_rate_v2v3(struct snd_usb_audio *chip, const struct audioformat *fmt, int rate) { int cur_rate, prev_rate; int clock; /* First, try to find a valid clock. This may trigger * automatic clock selection if the current clock is not * valid. */ clock = snd_usb_clock_find_source(chip, fmt, true); if (clock < 0) { /* We did not find a valid clock, but that might be * because the current sample rate does not match an * external clock source. Try again without validation * and we will do another validation after setting the * rate. */ clock = snd_usb_clock_find_source(chip, fmt, false); /* Hardcoded sample rates */ if (chip->quirk_flags & QUIRK_FLAG_IGNORE_CLOCK_SOURCE) return 0; if (clock < 0) return clock; } prev_rate = get_sample_rate_v2v3(chip, fmt->iface, fmt->altsetting, clock); if (prev_rate == rate) goto validation; cur_rate = snd_usb_set_sample_rate_v2v3(chip, fmt, clock, rate); if (cur_rate < 0) { usb_audio_err(chip, "%d:%d: cannot set freq %d (v2/v3): err %d\n", fmt->iface, fmt->altsetting, rate, cur_rate); return cur_rate; } if (!cur_rate) cur_rate = prev_rate; if (cur_rate != rate) { usb_audio_dbg(chip, "%d:%d: freq mismatch: req %d, clock runs @%d\n", fmt->iface, fmt->altsetting, rate, cur_rate); /* continue processing */ } /* FIXME - TEAC devices require the immediate interface setup */ if (USB_ID_VENDOR(chip->usb_id) == 0x0644) { bool cur_base_48k = (rate % 48000 == 0); bool prev_base_48k = (prev_rate % 48000 == 0); if (cur_base_48k != prev_base_48k) { usb_set_interface(chip->dev, fmt->iface, fmt->altsetting); if (chip->quirk_flags & QUIRK_FLAG_IFACE_DELAY) msleep(50); } } validation: /* validate clock after rate change */ if (!uac_clock_source_is_valid(chip, fmt, clock)) return -ENXIO; return 0; } int snd_usb_init_sample_rate(struct snd_usb_audio *chip, const struct audioformat *fmt, int rate) { usb_audio_dbg(chip, "%d:%d Set sample rate %d, clock %d\n", fmt->iface, fmt->altsetting, rate, fmt->clock); switch (fmt->protocol) { case UAC_VERSION_1: default: return set_sample_rate_v1(chip, fmt, rate); case UAC_VERSION_3: if (chip->badd_profile >= UAC3_FUNCTION_SUBCLASS_GENERIC_IO) { if (rate != UAC3_BADD_SAMPLING_RATE) return -ENXIO; else return 0; } fallthrough; case UAC_VERSION_2: return set_sample_rate_v2v3(chip, fmt, rate); } } |
| 70 431 69 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 | /* SPDX-License-Identifier: GPL-2.0-or-later */ /* SCTP kernel implementation * (C) Copyright IBM Corp. 2001, 2004 * Copyright (c) 1999-2000 Cisco, Inc. * Copyright (c) 1999-2001 Motorola, Inc. * Copyright (c) 2001 Intel Corp. * * This file is part of the SCTP kernel implementation * * Please send any bug reports or fixes you make to the * email addresses: * lksctp developers <linux-sctp@vger.kernel.org> * * Written or modified by: * Randall Stewart <randall@sctp.chicago.il.us> * Ken Morneau <kmorneau@cisco.com> * Qiaobing Xie <qxie1@email.mot.com> * La Monte H.P. Yarroll <piggy@acm.org> * Karl Knutson <karl@athena.chicago.il.us> * Jon Grimm <jgrimm@us.ibm.com> * Xingang Guo <xingang.guo@intel.com> * Hui Huang <hui.huang@nokia.com> * Sridhar Samudrala <sri@us.ibm.com> * Daisy Chang <daisyc@us.ibm.com> * Dajiang Zhang <dajiang.zhang@nokia.com> * Ardelle Fan <ardelle.fan@intel.com> * Ryan Layer <rmlayer@us.ibm.com> * Anup Pemmaiah <pemmaiah@cc.usu.edu> * Kevin Gao <kevin.gao@intel.com> */ #ifndef __sctp_structs_h__ #define __sctp_structs_h__ #include <crypto/sha2.h> #include <linux/ktime.h> #include <linux/generic-radix-tree.h> #include <linux/rhashtable-types.h> #include <linux/socket.h> /* linux/in.h needs this!! */ #include <linux/in.h> /* We get struct sockaddr_in. */ #include <linux/in6.h> /* We get struct in6_addr */ #include <linux/ipv6.h> #include <asm/param.h> /* We get MAXHOSTNAMELEN. */ #include <linux/atomic.h> /* This gets us atomic counters. */ #include <linux/skbuff.h> /* We need sk_buff_head. */ #include <linux/workqueue.h> /* We need tq_struct. */ #include <linux/sctp.h> /* We need sctp* header structs. */ #include <net/sctp/auth.h> /* We need auth specific structs */ #include <net/ip.h> /* For inet_skb_parm */ /* A convenience structure for handling sockaddr structures. * We should wean ourselves off this. */ union sctp_addr { struct sockaddr_inet sa; /* Large enough for both address families */ struct sockaddr_in v4; struct sockaddr_in6 v6; }; /* Forward declarations for data structures. */ struct sctp_globals; struct sctp_endpoint; struct sctp_association; struct sctp_transport; struct sctp_packet; struct sctp_chunk; struct sctp_inq; struct sctp_outq; struct sctp_bind_addr; struct sctp_ulpq; struct sctp_ep_common; struct sctp_stream; #include <net/sctp/tsnmap.h> #include <net/sctp/ulpevent.h> #include <net/sctp/ulpqueue.h> #include <net/sctp/stream_interleave.h> /* Structures useful for managing bind/connect. */ struct sctp_bind_bucket { unsigned short port; signed char fastreuse; signed char fastreuseport; kuid_t fastuid; struct hlist_node node; struct hlist_head owner; struct net *net; }; struct sctp_bind_hashbucket { spinlock_t lock; struct hlist_head chain; }; /* Used for hashing all associations. */ struct sctp_hashbucket { rwlock_t lock; struct hlist_head chain; } __attribute__((__aligned__(8))); /* The SCTP globals structure. */ extern struct sctp_globals { /* This is a list of groups of functions for each address * family that we support. */ struct list_head address_families; /* This is the hash of all endpoints. */ struct sctp_hashbucket *ep_hashtable; /* This is the sctp port control hash. */ struct sctp_bind_hashbucket *port_hashtable; /* This is the hash of all transports. */ struct rhltable transport_hashtable; /* Sizes of above hashtables. */ int ep_hashsize; int port_hashsize; /* Default initialization values to be applied to new associations. */ __u16 max_instreams; __u16 max_outstreams; /* Flag to indicate whether computing and verifying checksum * is disabled. */ bool checksum_disable; } sctp_globals; #define sctp_max_instreams (sctp_globals.max_instreams) #define sctp_max_outstreams (sctp_globals.max_outstreams) #define sctp_address_families (sctp_globals.address_families) #define sctp_ep_hashsize (sctp_globals.ep_hashsize) #define sctp_ep_hashtable (sctp_globals.ep_hashtable) #define sctp_port_hashsize (sctp_globals.port_hashsize) #define sctp_port_hashtable (sctp_globals.port_hashtable) #define sctp_transport_hashtable (sctp_globals.transport_hashtable) #define sctp_checksum_disable (sctp_globals.checksum_disable) /* SCTP Socket type: UDP or TCP style. */ enum sctp_socket_type { SCTP_SOCKET_UDP = 0, SCTP_SOCKET_UDP_HIGH_BANDWIDTH, SCTP_SOCKET_TCP }; /* Per socket SCTP information. */ struct sctp_sock { /* inet_sock has to be the first member of sctp_sock */ struct inet_sock inet; /* What kind of a socket is this? */ enum sctp_socket_type type; /* PF_ family specific functions. */ struct sctp_pf *pf; /* What is our base endpointer? */ struct sctp_endpoint *ep; struct sctp_bind_bucket *bind_hash; /* Various Socket Options. */ __u16 default_stream; __u32 default_ppid; __u16 default_flags; __u32 default_context; __u32 default_timetolive; __u32 default_rcv_context; int max_burst; /* Heartbeat interval: The endpoint sends out a Heartbeat chunk to * the destination address every heartbeat interval. This value * will be inherited by all new associations. */ __u32 hbinterval; __u32 probe_interval; __be16 udp_port; __be16 encap_port; /* This is the max_retrans value for new associations. */ __u16 pathmaxrxt; __u32 flowlabel; __u8 dscp; __u16 pf_retrans; __u16 ps_retrans; /* The initial Path MTU to use for new associations. */ __u32 pathmtu; /* The default SACK delay timeout for new associations. */ __u32 sackdelay; __u32 sackfreq; /* Flags controlling Heartbeat, SACK delay, and Path MTU Discovery. */ __u32 param_flags; __u32 default_ss; struct sctp_rtoinfo rtoinfo; struct sctp_paddrparams paddrparam; struct sctp_assocparams assocparams; /* * These two structures must be grouped together for the usercopy * whitelist region. */ __u16 subscribe; struct sctp_initmsg initmsg; int user_frag; __u32 autoclose; __u32 adaptation_ind; __u32 pd_point; __u16 nodelay:1, pf_expose:2, reuse:1, disable_fragments:1, v4mapped:1, frag_interleave:1, recvrcvinfo:1, recvnxtinfo:1, data_ready_signalled:1, cookie_auth_enable:1; atomic_t pd_mode; /* Fields after this point will be skipped on copies, like on accept * and peeloff operations */ /* Receive to here while partial delivery is in effect. */ struct sk_buff_head pd_lobby; struct list_head auto_asconf_list; int do_auto_asconf; }; #define sctp_sk(ptr) container_of_const(ptr, struct sctp_sock, inet.sk) static inline struct sock *sctp_opt2sk(const struct sctp_sock *sp) { return (struct sock *)sp; } #if IS_ENABLED(CONFIG_IPV6) struct sctp6_sock { struct sctp_sock sctp; struct ipv6_pinfo inet6; }; #endif /* CONFIG_IPV6 */ /* This is our APPLICATION-SPECIFIC state cookie. * THIS IS NOT DICTATED BY THE SPECIFICATION. */ /* These are the parts of an association which we send in the cookie. * Most of these are straight out of: * RFC2960 12.2 Parameters necessary per association (i.e. the TCB) * */ struct sctp_cookie { /* My : Tag expected in every inbound packet and sent * Verification: in the INIT or INIT ACK chunk. * Tag : */ __u32 my_vtag; /* Peer's : Tag expected in every outbound packet except * Verification: in the INIT chunk. * Tag : */ __u32 peer_vtag; /* The rest of these are not from the spec, but really need to * be in the cookie. */ /* My Tie Tag : Assist in discovering a restarting association. */ __u32 my_ttag; /* Peer's Tie Tag: Assist in discovering a restarting association. */ __u32 peer_ttag; /* When does this cookie expire? */ ktime_t expiration; /* Number of inbound/outbound streams which are set * and negotiated during the INIT process. */ __u16 sinit_num_ostreams; __u16 sinit_max_instreams; /* This is the first sequence number I used. */ __u32 initial_tsn; /* This holds the originating address of the INIT packet. */ union sctp_addr peer_addr; /* IG Section 2.35.3 * Include the source port of the INIT-ACK */ __u16 my_port; __u8 prsctp_capable; /* Padding for future use */ __u8 padding; __u32 adaptation_ind; __u8 auth_random[sizeof(struct sctp_paramhdr) + SCTP_AUTH_RANDOM_LENGTH]; __u8 auth_hmacs[SCTP_AUTH_NUM_HMACS * sizeof(__u16) + 2]; __u8 auth_chunks[sizeof(struct sctp_paramhdr) + SCTP_AUTH_MAX_CHUNKS]; /* This is a shim for my peer's INIT packet, followed by * a copy of the raw address list of the association. * The length of the raw address list is saved in the * raw_addr_list_len field, which will be used at the time when * the association TCB is re-constructed from the cookie. */ __u32 raw_addr_list_len; /* struct sctp_init_chunk peer_init[]; */ }; /* The format of our cookie that we send to our peer. */ struct sctp_signed_cookie { __u8 mac[SCTP_COOKIE_MAC_SIZE]; __u32 __pad; /* force sctp_cookie alignment to 64 bits */ struct sctp_cookie c; } __packed; /* This is another convenience type to allocate memory for address * params for the maximum size and pass such structures around * internally. */ union sctp_addr_param { struct sctp_paramhdr p; struct sctp_ipv4addr_param v4; struct sctp_ipv6addr_param v6; }; /* A convenience type to allow walking through the various * parameters and avoid casting all over the place. */ union sctp_params { void *v; struct sctp_paramhdr *p; struct sctp_cookie_preserve_param *life; struct sctp_hostname_param *dns; struct sctp_cookie_param *cookie; struct sctp_supported_addrs_param *sat; struct sctp_ipv4addr_param *v4; struct sctp_ipv6addr_param *v6; union sctp_addr_param *addr; struct sctp_adaptation_ind_param *aind; struct sctp_supported_ext_param *ext; struct sctp_random_param *random; struct sctp_chunks_param *chunks; struct sctp_hmac_algo_param *hmac_algo; struct sctp_addip_param *addip; }; /* RFC 2960. Section 3.3.5 Heartbeat. * Heartbeat Information: variable length * The Sender-specific Heartbeat Info field should normally include * information about the sender's current time when this HEARTBEAT * chunk is sent and the destination transport address to which this * HEARTBEAT is sent (see Section 8.3). */ struct sctp_sender_hb_info { struct sctp_paramhdr param_hdr; union sctp_addr daddr; unsigned long sent_at; __u64 hb_nonce; __u32 probe_size; }; int sctp_stream_init(struct sctp_stream *stream, __u16 outcnt, __u16 incnt, gfp_t gfp); int sctp_stream_init_ext(struct sctp_stream *stream, __u16 sid); void sctp_stream_free(struct sctp_stream *stream); void sctp_stream_clear(struct sctp_stream *stream); void sctp_stream_update(struct sctp_stream *stream, struct sctp_stream *new); /* What is the current SSN number for this stream? */ #define sctp_ssn_peek(stream, type, sid) \ (sctp_stream_##type((stream), (sid))->ssn) /* Return the next SSN number for this stream. */ #define sctp_ssn_next(stream, type, sid) \ (sctp_stream_##type((stream), (sid))->ssn++) /* Skip over this ssn and all below. */ #define sctp_ssn_skip(stream, type, sid, ssn) \ (sctp_stream_##type((stream), (sid))->ssn = ssn + 1) /* What is the current MID number for this stream? */ #define sctp_mid_peek(stream, type, sid) \ (sctp_stream_##type((stream), (sid))->mid) /* Return the next MID number for this stream. */ #define sctp_mid_next(stream, type, sid) \ (sctp_stream_##type((stream), (sid))->mid++) /* Skip over this mid and all below. */ #define sctp_mid_skip(stream, type, sid, mid) \ (sctp_stream_##type((stream), (sid))->mid = mid + 1) /* What is the current MID_uo number for this stream? */ #define sctp_mid_uo_peek(stream, type, sid) \ (sctp_stream_##type((stream), (sid))->mid_uo) /* Return the next MID_uo number for this stream. */ #define sctp_mid_uo_next(stream, type, sid) \ (sctp_stream_##type((stream), (sid))->mid_uo++) /* * Pointers to address related SCTP functions. * (i.e. things that depend on the address family.) */ struct sctp_af { int (*sctp_xmit) (struct sk_buff *skb, struct sctp_transport *); int (*setsockopt) (struct sock *sk, int level, int optname, sockptr_t optval, unsigned int optlen); int (*getsockopt) (struct sock *sk, int level, int optname, char __user *optval, int __user *optlen); void (*get_dst) (struct sctp_transport *t, union sctp_addr *saddr, struct flowi *fl, struct sock *sk); void (*get_saddr) (struct sctp_sock *sk, struct sctp_transport *t, struct flowi *fl); void (*copy_addrlist) (struct list_head *, struct net_device *); int (*cmp_addr) (const union sctp_addr *addr1, const union sctp_addr *addr2); void (*addr_copy) (union sctp_addr *dst, union sctp_addr *src); void (*from_skb) (union sctp_addr *, struct sk_buff *skb, int saddr); void (*from_sk) (union sctp_addr *, struct sock *sk); bool (*from_addr_param) (union sctp_addr *, union sctp_addr_param *, __be16 port, int iif); int (*to_addr_param) (const union sctp_addr *, union sctp_addr_param *); int (*addr_valid) (union sctp_addr *, struct sctp_sock *, const struct sk_buff *); enum sctp_scope (*scope)(union sctp_addr *); void (*inaddr_any) (union sctp_addr *, __be16); int (*is_any) (const union sctp_addr *); int (*available) (union sctp_addr *, struct sctp_sock *); int (*skb_iif) (const struct sk_buff *sk); int (*skb_sdif)(const struct sk_buff *sk); int (*is_ce) (const struct sk_buff *sk); void (*seq_dump_addr)(struct seq_file *seq, union sctp_addr *addr); void (*ecn_capable)(struct sock *sk); __u16 net_header_len; int sockaddr_len; int (*ip_options_len)(struct sock *sk); sa_family_t sa_family; struct list_head list; }; struct sctp_af *sctp_get_af_specific(sa_family_t); int sctp_register_af(struct sctp_af *); /* Protocol family functions. */ struct sctp_pf { void (*event_msgname)(struct sctp_ulpevent *, char *, int *); void (*skb_msgname) (struct sk_buff *, char *, int *); int (*af_supported) (sa_family_t, struct sctp_sock *); int (*cmp_addr) (const union sctp_addr *, const union sctp_addr *, struct sctp_sock *); int (*bind_verify) (struct sctp_sock *, union sctp_addr *); int (*send_verify) (struct sctp_sock *, union sctp_addr *); int (*supported_addrs)(const struct sctp_sock *, __be16 *); struct sock *(*create_accept_sk) (struct sock *sk, struct sctp_association *asoc, bool kern); int (*addr_to_user)(struct sctp_sock *sk, union sctp_addr *addr); void (*to_sk_saddr)(union sctp_addr *, struct sock *sk); void (*to_sk_daddr)(union sctp_addr *, struct sock *sk); void (*copy_ip_options)(struct sock *sk, struct sock *newsk); struct sctp_af *af; }; /* Structure to track chunk fragments that have been acked, but peer * fragments of the same message have not. */ struct sctp_datamsg { /* Chunks waiting to be submitted to lower layer. */ struct list_head chunks; /* Reference counting. */ refcount_t refcnt; /* When is this message no longer interesting to the peer? */ unsigned long expires_at; /* Did the message fail to send? */ int send_error; u8 send_failed:1, can_delay:1, /* should this message be Nagle delayed */ abandoned:1; /* should this message be abandoned */ }; struct sctp_datamsg *sctp_datamsg_from_user(struct sctp_association *, struct sctp_sndrcvinfo *, struct iov_iter *); void sctp_datamsg_free(struct sctp_datamsg *); void sctp_datamsg_put(struct sctp_datamsg *); void sctp_chunk_fail(struct sctp_chunk *, int error); int sctp_chunk_abandoned(struct sctp_chunk *); /* RFC2960 1.4 Key Terms * * o Chunk: A unit of information within an SCTP packet, consisting of * a chunk header and chunk-specific content. * * As a matter of convenience, we remember the SCTP common header for * each chunk as well as a few other header pointers... */ struct sctp_chunk { struct list_head list; refcount_t refcnt; /* How many times this chunk have been sent, for prsctp RTX policy */ int sent_count; union { /* This is our link to the per-transport transmitted list. */ struct list_head transmitted_list; /* List in specific stream outq */ struct list_head stream_list; }; /* This field is used by chunks that hold fragmented data. * For the first fragment this is the list that holds the rest of * fragments. For the remaining fragments, this is the link to the * frag_list maintained in the first fragment. */ struct list_head frag_list; /* This points to the sk_buff containing the actual data. */ struct sk_buff *skb; union { /* In case of GSO packets, this will store the head one */ struct sk_buff *head_skb; /* In case of auth enabled, this will point to the shkey */ struct sctp_shared_key *shkey; }; /* These are the SCTP headers by reverse order in a packet. * Note that some of these may happen more than once. In that * case, we point at the "current" one, whatever that means * for that level of header. */ /* We point this at the FIRST TLV parameter to chunk_hdr. */ union sctp_params param_hdr; union { __u8 *v; struct sctp_datahdr *data_hdr; struct sctp_inithdr *init_hdr; struct sctp_sackhdr *sack_hdr; struct sctp_heartbeathdr *hb_hdr; struct sctp_sender_hb_info *hbs_hdr; struct sctp_shutdownhdr *shutdown_hdr; struct sctp_signed_cookie *cookie_hdr; struct sctp_ecnehdr *ecne_hdr; struct sctp_cwrhdr *ecn_cwr_hdr; struct sctp_errhdr *err_hdr; struct sctp_addiphdr *addip_hdr; struct sctp_fwdtsn_hdr *fwdtsn_hdr; struct sctp_authhdr *auth_hdr; struct sctp_idatahdr *idata_hdr; struct sctp_ifwdtsn_hdr *ifwdtsn_hdr; } subh; __u8 *chunk_end; struct sctp_chunkhdr *chunk_hdr; struct sctphdr *sctp_hdr; /* This needs to be recoverable for SCTP_SEND_FAILED events. */ struct sctp_sndrcvinfo sinfo; /* Which association does this belong to? */ struct sctp_association *asoc; /* What endpoint received this chunk? */ struct sctp_ep_common *rcvr; /* We fill this in if we are calculating RTT. */ unsigned long sent_at; /* What is the origin IP address for this chunk? */ union sctp_addr source; /* Destination address for this chunk. */ union sctp_addr dest; /* For outbound message, track all fragments for SEND_FAILED. */ struct sctp_datamsg *msg; /* For an inbound chunk, this tells us where it came from. * For an outbound chunk, it tells us where we'd like it to * go. It is NULL if we have no preference. */ struct sctp_transport *transport; /* SCTP-AUTH: For the special case inbound processing of COOKIE-ECHO * we need save a pointer to the AUTH chunk, since the SCTP-AUTH * spec violates the principle premis that all chunks are processed * in order. */ struct sk_buff *auth_chunk; #define SCTP_CAN_FRTX 0x0 #define SCTP_NEED_FRTX 0x1 #define SCTP_DONT_FRTX 0x2 __u16 rtt_in_progress:1, /* This chunk used for RTT calc? */ has_tsn:1, /* Does this chunk have a TSN yet? */ has_ssn:1, /* Does this chunk have a SSN yet? */ #define has_mid has_ssn singleton:1, /* Only chunk in the packet? */ end_of_packet:1, /* Last chunk in the packet? */ ecn_ce_done:1, /* Have we processed the ECN CE bit? */ pdiscard:1, /* Discard the whole packet now? */ tsn_gap_acked:1, /* Is this chunk acked by a GAP ACK? */ data_accepted:1, /* At least 1 chunk accepted */ auth:1, /* IN: was auth'ed | OUT: needs auth */ has_asconf:1, /* IN: have seen an asconf before */ pmtu_probe:1, /* Used by PLPMTUD, can be set in s HB chunk */ tsn_missing_report:2, /* Data chunk missing counter. */ fast_retransmit:2; /* Is this chunk fast retransmitted? */ }; #define sctp_chunk_retransmitted(chunk) (chunk->sent_count > 1) void sctp_chunk_hold(struct sctp_chunk *); void sctp_chunk_put(struct sctp_chunk *); int sctp_user_addto_chunk(struct sctp_chunk *chunk, int len, struct iov_iter *from); void sctp_chunk_free(struct sctp_chunk *); void *sctp_addto_chunk(struct sctp_chunk *, int len, const void *data); struct sctp_chunk *sctp_chunkify(struct sk_buff *, const struct sctp_association *, struct sock *, gfp_t gfp); void sctp_init_addrs(struct sctp_chunk *, union sctp_addr *, union sctp_addr *); const union sctp_addr *sctp_source(const struct sctp_chunk *chunk); static inline __u16 sctp_chunk_stream_no(struct sctp_chunk *ch) { return ntohs(ch->subh.data_hdr->stream); } enum { SCTP_ADDR_NEW, /* new address added to assoc/ep */ SCTP_ADDR_SRC, /* address can be used as source */ SCTP_ADDR_DEL, /* address about to be deleted */ }; /* This is a structure for holding either an IPv6 or an IPv4 address. */ struct sctp_sockaddr_entry { struct list_head list; struct rcu_head rcu; union sctp_addr a; __u8 state; __u8 valid; }; #define SCTP_ADDRESS_TICK_DELAY 500 /* This structure holds lists of chunks as we are assembling for * transmission. */ struct sctp_packet { /* These are the SCTP header values (host order) for the packet. */ __u16 source_port; __u16 destination_port; __u32 vtag; /* This contains the payload chunks. */ struct list_head chunk_list; /* This is the overhead of the sctp and ip headers. */ size_t overhead; /* This is the total size of all chunks INCLUDING padding. */ size_t size; /* This is the maximum size this packet may have */ size_t max_size; /* The packet is destined for this transport address. * The function we finally use to pass down to the next lower * layer lives in the transport structure. */ struct sctp_transport *transport; /* pointer to the auth chunk for this packet */ struct sctp_chunk *auth; u8 has_cookie_echo:1, /* This packet contains a COOKIE-ECHO chunk. */ has_sack:1, /* This packet contains a SACK chunk. */ has_auth:1, /* This packet contains an AUTH chunk */ has_data:1, /* This packet contains at least 1 DATA chunk */ ipfragok:1; /* So let ip fragment this packet */ }; void sctp_packet_init(struct sctp_packet *, struct sctp_transport *, __u16 sport, __u16 dport); void sctp_packet_config(struct sctp_packet *, __u32 vtag, int); enum sctp_xmit sctp_packet_transmit_chunk(struct sctp_packet *packet, struct sctp_chunk *chunk, int one_packet, gfp_t gfp); enum sctp_xmit sctp_packet_append_chunk(struct sctp_packet *packet, struct sctp_chunk *chunk); int sctp_packet_transmit(struct sctp_packet *, gfp_t); void sctp_packet_free(struct sctp_packet *); static inline int sctp_packet_empty(struct sctp_packet *packet) { return packet->size == packet->overhead; } /* This represents a remote transport address. * For local transport addresses, we just use union sctp_addr. * * RFC2960 Section 1.4 Key Terms * * o Transport address: A Transport Address is traditionally defined * by Network Layer address, Transport Layer protocol and Transport * Layer port number. In the case of SCTP running over IP, a * transport address is defined by the combination of an IP address * and an SCTP port number (where SCTP is the Transport protocol). * * RFC2960 Section 7.1 SCTP Differences from TCP Congestion control * * o The sender keeps a separate congestion control parameter set for * each of the destination addresses it can send to (not each * source-destination pair but for each destination). The parameters * should decay if the address is not used for a long enough time * period. * */ struct sctp_transport { /* A list of transports. */ struct list_head transports; struct rhlist_head node; /* Reference counting. */ refcount_t refcnt; __u32 dead:1, /* RTO-Pending : A flag used to track if one of the DATA * chunks sent to this address is currently being * used to compute a RTT. If this flag is 0, * the next DATA chunk sent to this destination * should be used to compute a RTT and this flag * should be set. Every time the RTT * calculation completes (i.e. the DATA chunk * is SACK'd) clear this flag. */ rto_pending:1, /* * hb_sent : a flag that signals that we have a pending * heartbeat. */ hb_sent:1, /* Is the Path MTU update pending on this transport */ pmtu_pending:1, dst_pending_confirm:1, /* need to confirm neighbour */ /* Has this transport moved the ctsn since we last sacked */ sack_generation:1; u32 dst_cookie; struct flowi fl; /* This is the peer's IP address and port. */ union sctp_addr ipaddr; /* These are the functions we call to handle LLP stuff. */ struct sctp_af *af_specific; /* Which association do we belong to? */ struct sctp_association *asoc; /* RFC2960 * * 12.3 Per Transport Address Data * * For each destination transport address in the peer's * address list derived from the INIT or INIT ACK chunk, a * number of data elements needs to be maintained including: */ /* RTO : The current retransmission timeout value. */ unsigned long rto; __u32 rtt; /* This is the most recent RTT. */ /* RTTVAR : The current RTT variation. */ __u32 rttvar; /* SRTT : The current smoothed round trip time. */ __u32 srtt; /* * These are the congestion stats. */ /* cwnd : The current congestion window. */ __u32 cwnd; /* This is the actual cwnd. */ /* ssthresh : The current slow start threshold value. */ __u32 ssthresh; /* partial : The tracking method for increase of cwnd when in * bytes acked : congestion avoidance mode (see Section 6.2.2) */ __u32 partial_bytes_acked; /* Data that has been sent, but not acknowledged. */ __u32 flight_size; __u32 burst_limited; /* Holds old cwnd when max.burst is applied */ /* Destination */ struct dst_entry *dst; /* Source address. */ union sctp_addr saddr; /* Heartbeat interval: The endpoint sends out a Heartbeat chunk to * the destination address every heartbeat interval. */ unsigned long hbinterval; unsigned long probe_interval; /* SACK delay timeout */ unsigned long sackdelay; __u32 sackfreq; atomic_t mtu_info; /* When was the last time that we heard from this transport? We use * this to pick new active and retran paths. */ ktime_t last_time_heard; /* When was the last time that we sent a chunk using this * transport? We use this to check for idle transports */ unsigned long last_time_sent; /* Last time(in jiffies) when cwnd is reduced due to the congestion * indication based on ECNE chunk. */ unsigned long last_time_ecne_reduced; __be16 encap_port; /* This is the max_retrans value for the transport and will * be initialized from the assocs value. This can be changed * using the SCTP_SET_PEER_ADDR_PARAMS socket option. */ __u16 pathmaxrxt; __u32 flowlabel; __u8 dscp; /* This is the partially failed retrans value for the transport * and will be initialized from the assocs value. This can be changed * using the SCTP_PEER_ADDR_THLDS socket option */ __u16 pf_retrans; /* Used for primary path switchover. */ __u16 ps_retrans; /* PMTU : The current known path MTU. */ __u32 pathmtu; /* Flags controlling Heartbeat, SACK delay, and Path MTU Discovery. */ __u32 param_flags; /* The number of times INIT has been sent on this transport. */ int init_sent_count; /* state : The current state of this destination, * : i.e. SCTP_ACTIVE, SCTP_INACTIVE, SCTP_UNKNOWN. */ int state; /* These are the error stats for this destination. */ /* Error count : The current error count for this destination. */ unsigned short error_count; /* Per : A timer used by each destination. * Destination : * Timer : * * [Everywhere else in the text this is called T3-rtx. -ed] */ struct timer_list T3_rtx_timer; /* Heartbeat timer is per destination. */ struct timer_list hb_timer; /* Timer to handle ICMP proto unreachable envets */ struct timer_list proto_unreach_timer; /* Timer to handler reconf chunk rtx */ struct timer_list reconf_timer; /* Timer to send a probe HB packet for PLPMTUD */ struct timer_list probe_timer; /* Since we're using per-destination retransmission timers * (see above), we're also using per-destination "transmitted" * queues. This probably ought to be a private struct * accessible only within the outqueue, but it's not, yet. */ struct list_head transmitted; /* We build bundle-able packets for this transport here. */ struct sctp_packet packet; /* This is the list of transports that have chunks to send. */ struct list_head send_ready; /* State information saved for SFR_CACC algorithm. The key * idea in SFR_CACC is to maintain state at the sender on a * per-destination basis when a changeover happens. * char changeover_active; * char cycling_changeover; * __u32 next_tsn_at_change; * char cacc_saw_newack; */ struct { /* An unsigned integer, which stores the next TSN to be * used by the sender, at the moment of changeover. */ __u32 next_tsn_at_change; /* A flag which indicates the occurrence of a changeover */ char changeover_active; /* A flag which indicates whether the change of primary is * the first switch to this destination address during an * active switch. */ char cycling_changeover; /* A temporary flag, which is used during the processing of * a SACK to estimate the causative TSN(s)'s group. */ char cacc_saw_newack; } cacc; struct { __u16 pmtu; __u16 probe_size; __u16 probe_high; __u8 probe_count; __u8 state; } pl; /* plpmtud related */ /* 64-bit random number sent with heartbeat. */ __u64 hb_nonce; struct rcu_head rcu; }; struct sctp_transport *sctp_transport_new(struct net *, const union sctp_addr *, gfp_t); void sctp_transport_set_owner(struct sctp_transport *, struct sctp_association *); void sctp_transport_route(struct sctp_transport *, union sctp_addr *, struct sctp_sock *); void sctp_transport_pmtu(struct sctp_transport *, struct sock *sk); void sctp_transport_free(struct sctp_transport *); void sctp_transport_reset_t3_rtx(struct sctp_transport *); void sctp_transport_reset_hb_timer(struct sctp_transport *); void sctp_transport_reset_reconf_timer(struct sctp_transport *transport); void sctp_transport_reset_probe_timer(struct sctp_transport *transport); void sctp_transport_reset_raise_timer(struct sctp_transport *transport); int sctp_transport_hold(struct sctp_transport *); void sctp_transport_put(struct sctp_transport *); void sctp_transport_update_rto(struct sctp_transport *, __u32); void sctp_transport_raise_cwnd(struct sctp_transport *, __u32, __u32); void sctp_transport_lower_cwnd(struct sctp_transport *t, enum sctp_lower_cwnd reason); void sctp_transport_burst_limited(struct sctp_transport *); void sctp_transport_burst_reset(struct sctp_transport *); unsigned long sctp_transport_timeout(struct sctp_transport *); void sctp_transport_reset(struct sctp_transport *t); bool sctp_transport_update_pmtu(struct sctp_transport *t, u32 pmtu); void sctp_transport_immediate_rtx(struct sctp_transport *); void sctp_transport_dst_release(struct sctp_transport *t); void sctp_transport_dst_confirm(struct sctp_transport *t); void sctp_transport_pl_send(struct sctp_transport *t); bool sctp_transport_pl_recv(struct sctp_transport *t); /* This is the structure we use to queue packets as they come into * SCTP. We write packets to it and read chunks from it. */ struct sctp_inq { /* This is actually a queue of sctp_chunk each * containing a partially decoded packet. */ struct list_head in_chunk_list; /* This is the packet which is currently off the in queue and is * being worked on through the inbound chunk processing. */ struct sctp_chunk *in_progress; /* This is the delayed task to finish delivering inbound * messages. */ struct work_struct immediate; }; void sctp_inq_init(struct sctp_inq *); void sctp_inq_free(struct sctp_inq *); void sctp_inq_push(struct sctp_inq *, struct sctp_chunk *packet); struct sctp_chunk *sctp_inq_pop(struct sctp_inq *); struct sctp_chunkhdr *sctp_inq_peek(struct sctp_inq *); void sctp_inq_set_th_handler(struct sctp_inq *, work_func_t); /* This is the structure we use to hold outbound chunks. You push * chunks in and they automatically pop out the other end as bundled * packets (it calls (*output_handler)()). * * This structure covers sections 6.3, 6.4, 6.7, 6.8, 6.10, 7., 8.1, * and 8.2 of the v13 draft. * * It handles retransmissions. The connection to the timeout portion * of the state machine is through sctp_..._timeout() and timeout_handler. * * If you feed it SACKs, it will eat them. * * If you give it big chunks, it will fragment them. * * It assigns TSN's to data chunks. This happens at the last possible * instant before transmission. * * When free()'d, it empties itself out via output_handler(). */ struct sctp_outq { struct sctp_association *asoc; /* Data pending that has never been transmitted. */ struct list_head out_chunk_list; /* Stream scheduler being used */ struct sctp_sched_ops *sched; unsigned int out_qlen; /* Total length of queued data chunks. */ /* Error of send failed, may used in SCTP_SEND_FAILED event. */ unsigned int error; /* These are control chunks we want to send. */ struct list_head control_chunk_list; /* These are chunks that have been sacked but are above the * CTSN, or cumulative tsn ack point. */ struct list_head sacked; /* Put chunks on this list to schedule them for * retransmission. */ struct list_head retransmit; /* Put chunks on this list to save them for FWD TSN processing as * they were abandoned. */ struct list_head abandoned; /* How many unackd bytes do we have in-flight? */ __u32 outstanding_bytes; /* Are we doing fast-rtx on this queue */ char fast_rtx; /* Corked? */ char cork; }; void sctp_outq_init(struct sctp_association *, struct sctp_outq *); void sctp_outq_teardown(struct sctp_outq *); void sctp_outq_free(struct sctp_outq*); void sctp_outq_tail(struct sctp_outq *, struct sctp_chunk *chunk, gfp_t); int sctp_outq_sack(struct sctp_outq *, struct sctp_chunk *); int sctp_outq_is_empty(const struct sctp_outq *); void sctp_retransmit(struct sctp_outq *q, struct sctp_transport *transport, enum sctp_retransmit_reason reason); void sctp_retransmit_mark(struct sctp_outq *, struct sctp_transport *, __u8); void sctp_outq_uncork(struct sctp_outq *, gfp_t gfp); void sctp_prsctp_prune(struct sctp_association *asoc, struct sctp_sndrcvinfo *sinfo, int msg_len); void sctp_generate_fwdtsn(struct sctp_outq *q, __u32 sack_ctsn); /* Uncork and flush an outqueue. */ static inline void sctp_outq_cork(struct sctp_outq *q) { q->cork = 1; } /* SCTP skb control block. * sctp_input_cb is currently used on rx and sock rx queue */ struct sctp_input_cb { union { struct inet_skb_parm h4; #if IS_ENABLED(CONFIG_IPV6) struct inet6_skb_parm h6; #endif } header; struct sctp_chunk *chunk; struct sctp_af *af; __be16 encap_port; }; #define SCTP_INPUT_CB(__skb) ((struct sctp_input_cb *)&((__skb)->cb[0])) struct sctp_output_cb { struct sk_buff *last; }; #define SCTP_OUTPUT_CB(__skb) ((struct sctp_output_cb *)&((__skb)->cb[0])) static inline const struct sk_buff *sctp_gso_headskb(const struct sk_buff *skb) { const struct sctp_chunk *chunk = SCTP_INPUT_CB(skb)->chunk; return chunk->head_skb ? : skb; } /* These bind address data fields common between endpoints and associations */ struct sctp_bind_addr { /* RFC 2960 12.1 Parameters necessary for the SCTP instance * * SCTP Port: The local SCTP port number the endpoint is * bound to. */ __u16 port; /* RFC 2960 12.1 Parameters necessary for the SCTP instance * * Address List: The list of IP addresses that this instance * has bound. This information is passed to one's * peer(s) in INIT and INIT ACK chunks. */ struct list_head address_list; }; void sctp_bind_addr_init(struct sctp_bind_addr *, __u16 port); void sctp_bind_addr_free(struct sctp_bind_addr *); int sctp_bind_addr_copy(struct net *net, struct sctp_bind_addr *dest, const struct sctp_bind_addr *src, enum sctp_scope scope, gfp_t gfp, int flags); int sctp_bind_addr_dup(struct sctp_bind_addr *dest, const struct sctp_bind_addr *src, gfp_t gfp); int sctp_add_bind_addr(struct sctp_bind_addr *, union sctp_addr *, int new_size, __u8 addr_state, gfp_t gfp); int sctp_del_bind_addr(struct sctp_bind_addr *, union sctp_addr *); int sctp_bind_addr_match(struct sctp_bind_addr *, const union sctp_addr *, struct sctp_sock *); int sctp_bind_addr_conflict(struct sctp_bind_addr *, const union sctp_addr *, struct sctp_sock *, struct sctp_sock *); int sctp_bind_addr_state(const struct sctp_bind_addr *bp, const union sctp_addr *addr); int sctp_bind_addrs_check(struct sctp_sock *sp, struct sctp_sock *sp2, int cnt2); union sctp_addr *sctp_find_unmatch_addr(struct sctp_bind_addr *bp, const union sctp_addr *addrs, int addrcnt, struct sctp_sock *opt); union sctp_params sctp_bind_addrs_to_raw(const struct sctp_bind_addr *bp, int *addrs_len, gfp_t gfp); int sctp_raw_to_bind_addrs(struct sctp_bind_addr *bp, __u8 *raw, int len, __u16 port, gfp_t gfp); enum sctp_scope sctp_scope(const union sctp_addr *addr); int sctp_in_scope(struct net *net, const union sctp_addr *addr, const enum sctp_scope scope); int sctp_is_any(struct sock *sk, const union sctp_addr *addr); int sctp_is_ep_boundall(struct sock *sk); /* What type of endpoint? */ enum sctp_endpoint_type { SCTP_EP_TYPE_SOCKET, SCTP_EP_TYPE_ASSOCIATION, }; /* * A common base class to bridge the implementation view of a * socket (usually listening) endpoint versus an association's * local endpoint. * This common structure is useful for several purposes: * 1) Common interface for lookup routines. * a) Subfunctions work for either endpoint or association * b) Single interface to lookup allows hiding the lookup lock rather * than acquiring it externally. * 2) Common interface for the inbound chunk handling/state machine. * 3) Common object handling routines for reference counting, etc. * 4) Disentangle association lookup from endpoint lookup, where we * do not have to find our endpoint to find our association. * */ struct sctp_ep_common { /* Runtime type information. What kind of endpoint is this? */ enum sctp_endpoint_type type; /* Some fields to help us manage this object. * refcnt - Reference count access to this object. * dead - Do not attempt to use this object. */ refcount_t refcnt; bool dead; /* What socket does this endpoint belong to? */ struct sock *sk; /* Cache netns and it won't change once set */ struct net *net; /* This is where we receive inbound chunks. */ struct sctp_inq inqueue; /* This substructure includes the defining parameters of the * endpoint: * bind_addr.port is our shared port number. * bind_addr.address_list is our set of local IP addresses. */ struct sctp_bind_addr bind_addr; }; /* RFC Section 1.4 Key Terms * * o SCTP endpoint: The logical sender/receiver of SCTP packets. On a * multi-homed host, an SCTP endpoint is represented to its peers as a * combination of a set of eligible destination transport addresses to * which SCTP packets can be sent and a set of eligible source * transport addresses from which SCTP packets can be received. * All transport addresses used by an SCTP endpoint must use the * same port number, but can use multiple IP addresses. A transport * address used by an SCTP endpoint must not be used by another * SCTP endpoint. In other words, a transport address is unique * to an SCTP endpoint. * * From an implementation perspective, each socket has one of these. * A TCP-style socket will have exactly one association on one of * these. An UDP-style socket will have multiple associations hanging * off one of these. */ struct sctp_endpoint { /* Common substructure for endpoint and association. */ struct sctp_ep_common base; /* Fields to help us manage our entries in the hash tables. */ struct hlist_node node; int hashent; /* Associations: A list of current associations and mappings * to the data consumers for each association. This * may be in the form of a hash table or other * implementation dependent structure. The data * consumers may be process identification * information such as file descriptors, named pipe * pointer, or table pointers dependent on how SCTP * is implemented. */ /* This is really a list of struct sctp_association entries. */ struct list_head asocs; /* Cookie authentication key used by this endpoint */ struct hmac_sha256_key cookie_auth_key; /* sendbuf acct. policy. */ __u32 sndbuf_policy; /* rcvbuf acct. policy. */ __u32 rcvbuf_policy; /* SCTP-AUTH: hmacs for the endpoint encoded into parameter */ struct sctp_hmac_algo_param *auth_hmacs_list; /* SCTP-AUTH: chunks to authenticate encoded into parameter */ struct sctp_chunks_param *auth_chunk_list; /* SCTP-AUTH: endpoint shared keys */ struct list_head endpoint_shared_keys; __u16 active_key_id; __u8 ecn_enable:1, auth_enable:1, intl_enable:1, prsctp_enable:1, asconf_enable:1, reconf_enable:1; __u8 strreset_enable; struct rcu_head rcu; }; /* Recover the outer endpoint structure. */ static inline struct sctp_endpoint *sctp_ep(struct sctp_ep_common *base) { struct sctp_endpoint *ep; ep = container_of(base, struct sctp_endpoint, base); return ep; } /* These are function signatures for manipulating endpoints. */ struct sctp_endpoint *sctp_endpoint_new(struct sock *, gfp_t); void sctp_endpoint_free(struct sctp_endpoint *); void sctp_endpoint_put(struct sctp_endpoint *); int sctp_endpoint_hold(struct sctp_endpoint *ep); void sctp_endpoint_add_asoc(struct sctp_endpoint *, struct sctp_association *); struct sctp_association *sctp_endpoint_lookup_assoc( const struct sctp_endpoint *ep, const union sctp_addr *paddr, struct sctp_transport **); bool sctp_endpoint_is_peeled_off(struct sctp_endpoint *ep, const union sctp_addr *paddr); struct sctp_endpoint *sctp_endpoint_is_match(struct sctp_endpoint *ep, struct net *net, const union sctp_addr *laddr, int dif, int sdif); bool sctp_has_association(struct net *net, const union sctp_addr *laddr, const union sctp_addr *paddr, int dif, int sdif); int sctp_verify_init(struct net *net, const struct sctp_endpoint *ep, const struct sctp_association *asoc, enum sctp_cid cid, struct sctp_init_chunk *peer_init, struct sctp_chunk *chunk, struct sctp_chunk **err_chunk); int sctp_process_init(struct sctp_association *, struct sctp_chunk *chunk, const union sctp_addr *peer, struct sctp_init_chunk *init, gfp_t gfp); __u32 sctp_generate_tag(const struct sctp_endpoint *); __u32 sctp_generate_tsn(const struct sctp_endpoint *); struct sctp_inithdr_host { __u32 init_tag; __u32 a_rwnd; __u16 num_outbound_streams; __u16 num_inbound_streams; __u32 initial_tsn; }; struct sctp_stream_priorities { /* List of priorities scheduled */ struct list_head prio_sched; /* List of streams scheduled */ struct list_head active; /* The next stream in line */ struct sctp_stream_out_ext *next; __u16 prio; __u16 users; }; struct sctp_stream_out_ext { __u64 abandoned_unsent[SCTP_PR_INDEX(MAX) + 1]; __u64 abandoned_sent[SCTP_PR_INDEX(MAX) + 1]; struct list_head outq; /* chunks enqueued by this stream */ union { struct { /* Scheduled streams list */ struct list_head prio_list; struct sctp_stream_priorities *prio_head; }; /* Fields used by RR scheduler */ struct { struct list_head rr_list; }; struct { struct list_head fc_list; __u32 fc_length; __u16 fc_weight; }; }; }; struct sctp_stream_out { union { __u32 mid; __u16 ssn; }; __u32 mid_uo; struct sctp_stream_out_ext *ext; __u8 state; }; struct sctp_stream_in { union { __u32 mid; __u16 ssn; }; __u32 mid_uo; __u32 fsn; __u32 fsn_uo; char pd_mode; char pd_mode_uo; }; struct sctp_stream { GENRADIX(struct sctp_stream_out) out; GENRADIX(struct sctp_stream_in) in; __u16 outcnt; __u16 incnt; /* Current stream being sent, if any */ struct sctp_stream_out *out_curr; union { /* Fields used by priority scheduler */ struct { /* List of priorities scheduled */ struct list_head prio_list; }; /* Fields used by RR scheduler */ struct { /* List of streams scheduled */ struct list_head rr_list; /* The next stream in line */ struct sctp_stream_out_ext *rr_next; }; struct { struct list_head fc_list; }; }; struct sctp_stream_interleave *si; }; static inline struct sctp_stream_out *sctp_stream_out( struct sctp_stream *stream, __u16 sid) { return genradix_ptr(&stream->out, sid); } static inline struct sctp_stream_in *sctp_stream_in( struct sctp_stream *stream, __u16 sid) { return genradix_ptr(&stream->in, sid); } #define SCTP_SO(s, i) sctp_stream_out((s), (i)) #define SCTP_SI(s, i) sctp_stream_in((s), (i)) #define SCTP_STREAM_CLOSED 0x00 #define SCTP_STREAM_OPEN 0x01 static inline __u16 sctp_datachk_len(const struct sctp_stream *stream) { return stream->si->data_chunk_len; } static inline __u16 sctp_datahdr_len(const struct sctp_stream *stream) { return stream->si->data_chunk_len - sizeof(struct sctp_chunkhdr); } static inline __u16 sctp_ftsnchk_len(const struct sctp_stream *stream) { return stream->si->ftsn_chunk_len; } static inline __u16 sctp_ftsnhdr_len(const struct sctp_stream *stream) { return stream->si->ftsn_chunk_len - sizeof(struct sctp_chunkhdr); } /* SCTP_GET_ASSOC_STATS counters */ struct sctp_priv_assoc_stats { /* Maximum observed rto in the association during subsequent * observations. Value is set to 0 if no RTO measurement took place * The transport where the max_rto was observed is returned in * obs_rto_ipaddr */ struct sockaddr_storage obs_rto_ipaddr; __u64 max_obs_rto; /* Total In and Out SACKs received and sent */ __u64 isacks; __u64 osacks; /* Total In and Out packets received and sent */ __u64 opackets; __u64 ipackets; /* Total retransmitted chunks */ __u64 rtxchunks; /* TSN received > next expected */ __u64 outofseqtsns; /* Duplicate Chunks received */ __u64 idupchunks; /* Gap Ack Blocks received */ __u64 gapcnt; /* Unordered data chunks sent and received */ __u64 ouodchunks; __u64 iuodchunks; /* Ordered data chunks sent and received */ __u64 oodchunks; __u64 iodchunks; /* Control chunks sent and received */ __u64 octrlchunks; __u64 ictrlchunks; }; /* RFC2960 * * 12. Recommended Transmission Control Block (TCB) Parameters * * This section details a recommended set of parameters that should * be contained within the TCB for an implementation. This section is * for illustrative purposes and should not be deemed as requirements * on an implementation or as an exhaustive list of all parameters * inside an SCTP TCB. Each implementation may need its own additional * parameters for optimization. */ /* Here we have information about each individual association. */ struct sctp_association { /* A base structure common to endpoint and association. * In this context, it represents the associations's view * of the local endpoint of the association. */ struct sctp_ep_common base; /* Associations on the same socket. */ struct list_head asocs; /* association id. */ sctp_assoc_t assoc_id; /* This is our parent endpoint. */ struct sctp_endpoint *ep; /* These are those association elements needed in the cookie. */ struct sctp_cookie c; /* This is all information about our peer. */ struct { /* transport_addr_list * * Peer : A list of SCTP transport addresses that the * Transport : peer is bound to. This information is derived * Address : from the INIT or INIT ACK and is used to * List : associate an inbound packet with a given * : association. Normally this information is * : hashed or keyed for quick lookup and access * : of the TCB. * : The list is also initialized with the list * : of addresses passed with the sctp_connectx() * : call. * * It is a list of SCTP_transport's. */ struct list_head transport_addr_list; /* rwnd * * Peer Rwnd : Current calculated value of the peer's rwnd. */ __u32 rwnd; /* transport_count * * Peer : A count of the number of peer addresses * Transport : in the Peer Transport Address List. * Address : * Count : */ __u16 transport_count; /* port * The transport layer port number. */ __u16 port; /* primary_path * * Primary : This is the current primary destination * Path : transport address of the peer endpoint. It * : may also specify a source transport address * : on this endpoint. * * All of these paths live on transport_addr_list. * * At the bakeoffs, we discovered that the intent of * primaryPath is that it only changes when the ULP * asks to have it changed. We add the activePath to * designate the connection we are currently using to * transmit new data and most control chunks. */ struct sctp_transport *primary_path; /* Cache the primary path address here, when we * need a an address for msg_name. */ union sctp_addr primary_addr; /* active_path * The path that we are currently using to * transmit new data and most control chunks. */ struct sctp_transport *active_path; /* retran_path * * RFC2960 6.4 Multi-homed SCTP Endpoints * ... * Furthermore, when its peer is multi-homed, an * endpoint SHOULD try to retransmit a chunk to an * active destination transport address that is * different from the last destination address to * which the DATA chunk was sent. */ struct sctp_transport *retran_path; /* Pointer to last transport I have sent on. */ struct sctp_transport *last_sent_to; /* This is the last transport I have received DATA on. */ struct sctp_transport *last_data_from; /* * Mapping An array of bits or bytes indicating which out of * Array order TSN's have been received (relative to the * Last Rcvd TSN). If no gaps exist, i.e. no out of * order packets have been received, this array * will be set to all zero. This structure may be * in the form of a circular buffer or bit array. * * Last Rcvd : This is the last TSN received in * TSN : sequence. This value is set initially by * : taking the peer's Initial TSN, received in * : the INIT or INIT ACK chunk, and subtracting * : one from it. * * Throughout most of the specification this is called the * "Cumulative TSN ACK Point". In this case, we * ignore the advice in 12.2 in favour of the term * used in the bulk of the text. This value is hidden * in tsn_map--we get it by calling sctp_tsnmap_get_ctsn(). */ struct sctp_tsnmap tsn_map; /* This mask is used to disable sending the ASCONF chunk * with specified parameter to peer. */ __be16 addip_disabled_mask; /* These are capabilities which our peer advertised. */ __u16 ecn_capable:1, /* Can peer do ECN? */ ipv4_address:1, /* Peer understands IPv4 addresses? */ ipv6_address:1, /* Peer understands IPv6 addresses? */ asconf_capable:1, /* Does peer support ADDIP? */ prsctp_capable:1, /* Can peer do PR-SCTP? */ reconf_capable:1, /* Can peer do RE-CONFIG? */ intl_capable:1, /* Can peer do INTERLEAVE */ auth_capable:1, /* Is peer doing SCTP-AUTH? */ /* sack_needed: * This flag indicates if the next received * packet is to be responded to with a * SACK. This is initialized to 0. When a packet * is received sack_cnt is incremented. If this value * reaches 2 or more, a SACK is sent and the * value is reset to 0. Note: This is used only * when no DATA chunks are received out of * order. When DATA chunks are out of order, * SACK's are not delayed (see Section 6). */ sack_needed:1, /* Do we need to sack the peer? */ sack_generation:1, zero_window_announced:1; __u32 sack_cnt; __u32 adaptation_ind; /* Adaptation Code point. */ struct sctp_inithdr_host i; void *cookie; int cookie_len; /* ADDIP Section 4.2 Upon reception of an ASCONF Chunk. * C1) ... "Peer-Serial-Number'. This value MUST be initialized to the * Initial TSN Value minus 1 */ __u32 addip_serial; /* SCTP-AUTH: We need to know pears random number, hmac list * and authenticated chunk list. All that is part of the * cookie and these are just pointers to those locations */ struct sctp_random_param *peer_random; struct sctp_chunks_param *peer_chunks; struct sctp_hmac_algo_param *peer_hmacs; } peer; /* State : A state variable indicating what state the * : association is in, i.e. COOKIE-WAIT, * : COOKIE-ECHOED, ESTABLISHED, SHUTDOWN-PENDING, * : SHUTDOWN-SENT, SHUTDOWN-RECEIVED, SHUTDOWN-ACK-SENT. * * Note: No "CLOSED" state is illustrated since if a * association is "CLOSED" its TCB SHOULD be removed. * * In this implementation we DO have a CLOSED * state which is used during initiation and shutdown. * * State takes values from SCTP_STATE_*. */ enum sctp_state state; /* Overall : The overall association error count. * Error Count : [Clear this any time I get something.] */ int overall_error_count; /* The cookie life I award for any cookie. */ ktime_t cookie_life; /* These are the association's initial, max, and min RTO values. * These values will be initialized by system defaults, but can * be modified via the SCTP_RTOINFO socket option. */ unsigned long rto_initial; unsigned long rto_max; unsigned long rto_min; /* Maximum number of new data packets that can be sent in a burst. */ int max_burst; /* This is the max_retrans value for the association. This value will * be initialized from system defaults, but can be * modified by the SCTP_ASSOCINFO socket option. */ int max_retrans; /* This is the partially failed retrans value for the transport * and will be initialized from the assocs value. This can be * changed using the SCTP_PEER_ADDR_THLDS socket option */ __u16 pf_retrans; /* Used for primary path switchover. */ __u16 ps_retrans; /* Maximum number of times the endpoint will retransmit INIT */ __u16 max_init_attempts; /* How many times have we resent an INIT? */ __u16 init_retries; /* The largest timeout or RTO value to use in attempting an INIT */ unsigned long max_init_timeo; /* Heartbeat interval: The endpoint sends out a Heartbeat chunk to * the destination address every heartbeat interval. This value * will be inherited by all new transports. */ unsigned long hbinterval; unsigned long probe_interval; __be16 encap_port; /* This is the max_retrans value for new transports in the * association. */ __u16 pathmaxrxt; __u32 flowlabel; __u8 dscp; /* Flag that path mtu update is pending */ __u8 pmtu_pending; /* Association : The smallest PMTU discovered for all of the * PMTU : peer's transport addresses. */ __u32 pathmtu; /* Flags controlling Heartbeat, SACK delay, and Path MTU Discovery. */ __u32 param_flags; __u32 sackfreq; /* SACK delay timeout */ unsigned long sackdelay; unsigned long timeouts[SCTP_NUM_TIMEOUT_TYPES]; struct timer_list timers[SCTP_NUM_TIMEOUT_TYPES]; /* Transport to which SHUTDOWN chunk was last sent. */ struct sctp_transport *shutdown_last_sent_to; /* Transport to which INIT chunk was last sent. */ struct sctp_transport *init_last_sent_to; /* How many times have we resent a SHUTDOWN */ int shutdown_retries; /* Next TSN : The next TSN number to be assigned to a new * : DATA chunk. This is sent in the INIT or INIT * : ACK chunk to the peer and incremented each * : time a DATA chunk is assigned a TSN * : (normally just prior to transmit or during * : fragmentation). */ __u32 next_tsn; /* * Last Rcvd : This is the last TSN received in sequence. This value * TSN : is set initially by taking the peer's Initial TSN, * : received in the INIT or INIT ACK chunk, and * : subtracting one from it. * * Most of RFC 2960 refers to this as the Cumulative TSN Ack Point. */ __u32 ctsn_ack_point; /* PR-SCTP Advanced.Peer.Ack.Point */ __u32 adv_peer_ack_point; /* Highest TSN that is acknowledged by incoming SACKs. */ __u32 highest_sacked; /* TSN marking the fast recovery exit point */ __u32 fast_recovery_exit; /* Flag to track the current fast recovery state */ __u8 fast_recovery; /* The number of unacknowledged data chunks. Reported through * the SCTP_STATUS sockopt. */ __u16 unack_data; /* The total number of data chunks that we've had to retransmit * as the result of a T3 timer expiration */ __u32 rtx_data_chunks; /* This is the association's receive buffer space. This value is used * to set a_rwnd field in an INIT or a SACK chunk. */ __u32 rwnd; /* This is the last advertised value of rwnd over a SACK chunk. */ __u32 a_rwnd; /* Number of bytes by which the rwnd has slopped. The rwnd is allowed * to slop over a maximum of the association's frag_point. */ __u32 rwnd_over; /* Keeps treack of rwnd pressure. This happens when we have * a window, but not receive buffer (i.e small packets). This one * is releases slowly (1 PMTU at a time ). */ __u32 rwnd_press; /* This is the sndbuf size in use for the association. * This corresponds to the sndbuf size for the association, * as specified in the sk->sndbuf. */ int sndbuf_used; /* This is the amount of memory that this association has allocated * in the receive path at any given time. */ atomic_t rmem_alloc; /* This is the wait queue head for send requests waiting on * the association sndbuf space. */ wait_queue_head_t wait; /* The message size at which SCTP fragmentation will occur. */ __u32 frag_point; __u32 user_frag; /* Counter used to count INIT errors. */ int init_err_counter; /* Count the number of INIT cycles (for doubling timeout). */ int init_cycle; /* Default send parameters. */ __u16 default_stream; __u16 default_flags; __u32 default_ppid; __u32 default_context; __u32 default_timetolive; /* Default receive parameters */ __u32 default_rcv_context; /* Stream arrays */ struct sctp_stream stream; /* All outbound chunks go through this structure. */ struct sctp_outq outqueue; /* A smart pipe that will handle reordering and fragmentation, * as well as handle passing events up to the ULP. */ struct sctp_ulpq ulpq; /* Last TSN that caused an ECNE Chunk to be sent. */ __u32 last_ecne_tsn; /* Last TSN that caused a CWR Chunk to be sent. */ __u32 last_cwr_tsn; /* How many duplicated TSNs have we seen? */ int numduptsns; /* These are to support * "SCTP Extensions for Dynamic Reconfiguration of IP Addresses * and Enforcement of Flow and Message Limits" * <draft-ietf-tsvwg-addip-sctp-02.txt> * or "ADDIP" for short. */ /* ADDIP Section 4.1.1 Congestion Control of ASCONF Chunks * * R1) One and only one ASCONF Chunk MAY be in transit and * unacknowledged at any one time. If a sender, after sending * an ASCONF chunk, decides it needs to transfer another * ASCONF Chunk, it MUST wait until the ASCONF-ACK Chunk * returns from the previous ASCONF Chunk before sending a * subsequent ASCONF. Note this restriction binds each side, * so at any time two ASCONF may be in-transit on any given * association (one sent from each endpoint). * * [This is our one-and-only-one ASCONF in flight. If we do * not have an ASCONF in flight, this is NULL.] */ struct sctp_chunk *addip_last_asconf; /* ADDIP Section 5.2 Upon reception of an ASCONF Chunk. * * This is needed to implement items E1 - E4 of the updated * spec. Here is the justification: * * Since the peer may bundle multiple ASCONF chunks toward us, * we now need the ability to cache multiple ACKs. The section * describes in detail how they are cached and cleaned up. */ struct list_head asconf_ack_list; /* These ASCONF chunks are waiting to be sent. * * These chunks can't be pushed to outqueue until receiving * ASCONF_ACK for the previous ASCONF indicated by * addip_last_asconf, so as to guarantee that only one ASCONF * is in flight at any time. * * ADDIP Section 4.1.1 Congestion Control of ASCONF Chunks * * In defining the ASCONF Chunk transfer procedures, it is * essential that these transfers MUST NOT cause congestion * within the network. To achieve this, we place these * restrictions on the transfer of ASCONF Chunks: * * R1) One and only one ASCONF Chunk MAY be in transit and * unacknowledged at any one time. If a sender, after sending * an ASCONF chunk, decides it needs to transfer another * ASCONF Chunk, it MUST wait until the ASCONF-ACK Chunk * returns from the previous ASCONF Chunk before sending a * subsequent ASCONF. Note this restriction binds each side, * so at any time two ASCONF may be in-transit on any given * association (one sent from each endpoint). * * * [I really think this is EXACTLY the sort of intelligence * which already resides in sctp_outq. Please move this * queue and its supporting logic down there. --piggy] */ struct list_head addip_chunk_list; /* ADDIP Section 4.1 ASCONF Chunk Procedures * * A2) A serial number should be assigned to the Chunk. The * serial number SHOULD be a monotonically increasing * number. The serial number SHOULD be initialized at * the start of the association to the same value as the * Initial TSN and every time a new ASCONF chunk is created * it is incremented by one after assigning the serial number * to the newly created chunk. * * ADDIP * 3.1.1 Address/Stream Configuration Change Chunk (ASCONF) * * Serial Number : 32 bits (unsigned integer) * * This value represents a Serial Number for the ASCONF * Chunk. The valid range of Serial Number is from 0 to * 4294967295 (2^32 - 1). Serial Numbers wrap back to 0 * after reaching 4294967295. */ __u32 addip_serial; int src_out_of_asoc_ok; union sctp_addr *asconf_addr_del_pending; struct sctp_transport *new_transport; /* SCTP AUTH: list of the endpoint shared keys. These * keys are provided out of band by the user application * and can't change during the lifetime of the association */ struct list_head endpoint_shared_keys; /* SCTP AUTH: * The current generated association shared key (secret) */ struct sctp_auth_bytes *asoc_shared_key; struct sctp_shared_key *shkey; /* SCTP AUTH: hmac id of the first peer requested algorithm * that we support. */ __u16 default_hmac_id; __u16 active_key_id; __u8 need_ecne:1, /* Need to send an ECNE Chunk? */ temp:1, /* Is it a temporary association? */ pf_expose:2, /* Expose pf state? */ force_delay:1; __u8 strreset_enable; __u8 strreset_outstanding; /* request param count on the fly */ __u32 strreset_outseq; /* Update after receiving response */ __u32 strreset_inseq; /* Update after receiving request */ __u32 strreset_result[2]; /* save the results of last 2 responses */ struct sctp_chunk *strreset_chunk; /* save request chunk */ struct sctp_priv_assoc_stats stats; int sent_cnt_removable; __u16 subscribe; __u64 abandoned_unsent[SCTP_PR_INDEX(MAX) + 1]; __u64 abandoned_sent[SCTP_PR_INDEX(MAX) + 1]; /* Security identifiers from incoming (INIT). These are set by * security_sctp_assoc_request(). These will only be used by * SCTP TCP type sockets and peeled off connections as they * cause a new socket to be generated. security_sctp_sk_clone() * will then plug these into the new socket. */ u32 secid; u32 peer_secid; struct rcu_head rcu; }; /* An eyecatcher for determining if we are really looking at an * association data structure. */ enum { SCTP_ASSOC_EYECATCHER = 0xa550c123, }; /* Recover the outer association structure. */ static inline struct sctp_association *sctp_assoc(struct sctp_ep_common *base) { struct sctp_association *asoc; asoc = container_of(base, struct sctp_association, base); return asoc; } /* These are function signatures for manipulating associations. */ struct sctp_association * sctp_association_new(const struct sctp_endpoint *ep, const struct sock *sk, enum sctp_scope scope, gfp_t gfp); void sctp_association_free(struct sctp_association *); void sctp_association_put(struct sctp_association *); void sctp_association_hold(struct sctp_association *); struct sctp_transport *sctp_assoc_choose_alter_transport( struct sctp_association *, struct sctp_transport *); void sctp_assoc_update_retran_path(struct sctp_association *); struct sctp_transport *sctp_assoc_lookup_paddr(const struct sctp_association *, const union sctp_addr *); int sctp_assoc_lookup_laddr(struct sctp_association *asoc, const union sctp_addr *laddr); struct sctp_transport *sctp_assoc_add_peer(struct sctp_association *, const union sctp_addr *address, const gfp_t gfp, const int peer_state); void sctp_assoc_rm_peer(struct sctp_association *asoc, struct sctp_transport *peer); void sctp_assoc_control_transport(struct sctp_association *asoc, struct sctp_transport *transport, enum sctp_transport_cmd command, sctp_sn_error_t error); struct sctp_transport *sctp_assoc_lookup_tsn(struct sctp_association *, __u32); void sctp_assoc_migrate(struct sctp_association *, struct sock *); int sctp_assoc_update(struct sctp_association *old, struct sctp_association *new); __u32 sctp_association_get_next_tsn(struct sctp_association *); void sctp_assoc_update_frag_point(struct sctp_association *asoc); void sctp_assoc_set_pmtu(struct sctp_association *asoc, __u32 pmtu); void sctp_assoc_sync_pmtu(struct sctp_association *asoc); void sctp_assoc_rwnd_increase(struct sctp_association *, unsigned int); void sctp_assoc_rwnd_decrease(struct sctp_association *, unsigned int); void sctp_assoc_set_primary(struct sctp_association *, struct sctp_transport *); void sctp_assoc_del_nonprimary_peers(struct sctp_association *, struct sctp_transport *); int sctp_assoc_set_bind_addr_from_ep(struct sctp_association *asoc, enum sctp_scope scope, gfp_t gfp); int sctp_assoc_set_bind_addr_from_cookie(struct sctp_association *, struct sctp_cookie*, gfp_t gfp); int sctp_assoc_set_id(struct sctp_association *, gfp_t); void sctp_assoc_clean_asconf_ack_cache(const struct sctp_association *asoc); struct sctp_chunk *sctp_assoc_lookup_asconf_ack( const struct sctp_association *asoc, __be32 serial); void sctp_asconf_queue_teardown(struct sctp_association *asoc); int sctp_cmp_addr_exact(const union sctp_addr *ss1, const union sctp_addr *ss2); struct sctp_chunk *sctp_get_ecne_prepend(struct sctp_association *asoc); /* A convenience structure to parse out SCTP specific CMSGs. */ struct sctp_cmsgs { struct sctp_initmsg *init; struct sctp_sndrcvinfo *srinfo; struct sctp_sndinfo *sinfo; struct sctp_prinfo *prinfo; struct sctp_authinfo *authinfo; struct msghdr *addrs_msg; }; /* Structure for tracking memory objects */ struct sctp_dbg_objcnt_entry { char *label; atomic_t *counter; }; #endif /* __sctp_structs_h__ */ |
| 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 | // SPDX-License-Identifier: GPL-2.0-only /*************************************************************************** * Copyright (C) 2010-2012 by Bruno Prémont <bonbons@linux-vserver.org> * * * * Based on Logitech G13 driver (v0.4) * * Copyright (C) 2009 by Rick L. Vinyard, Jr. <rvinyard@cs.nmsu.edu> * * * ***************************************************************************/ #include <linux/hid.h> #include <linux/hid-debug.h> #include <linux/input.h> #include "hid-ids.h" #include <linux/fb.h> #include <linux/vmalloc.h> #include <linux/backlight.h> #include <linux/lcd.h> #include <linux/leds.h> #include <linux/seq_file.h> #include <linux/debugfs.h> #include <linux/completion.h> #include <linux/uaccess.h> #include <linux/module.h> #include "hid-picolcd.h" void picolcd_leds_set(struct picolcd_data *data) { struct hid_report *report; unsigned long flags; if (!data->led[0]) return; report = picolcd_out_report(REPORT_LED_STATE, data->hdev); if (!report || report->maxfield != 1 || report->field[0]->report_count != 1) return; spin_lock_irqsave(&data->lock, flags); hid_set_field(report->field[0], 0, data->led_state); if (!(data->status & PICOLCD_FAILED)) hid_hw_request(data->hdev, report, HID_REQ_SET_REPORT); spin_unlock_irqrestore(&data->lock, flags); } static void picolcd_led_set_brightness(struct led_classdev *led_cdev, enum led_brightness value) { struct device *dev; struct hid_device *hdev; struct picolcd_data *data; int i, state = 0; dev = led_cdev->dev->parent; hdev = to_hid_device(dev); data = hid_get_drvdata(hdev); if (!data) return; for (i = 0; i < 8; i++) { if (led_cdev != data->led[i]) continue; state = (data->led_state >> i) & 1; if (value == LED_OFF && state) { data->led_state &= ~(1 << i); picolcd_leds_set(data); } else if (value != LED_OFF && !state) { data->led_state |= 1 << i; picolcd_leds_set(data); } break; } } static enum led_brightness picolcd_led_get_brightness(struct led_classdev *led_cdev) { struct device *dev; struct hid_device *hdev; struct picolcd_data *data; int i, value = 0; dev = led_cdev->dev->parent; hdev = to_hid_device(dev); data = hid_get_drvdata(hdev); for (i = 0; i < 8; i++) if (led_cdev == data->led[i]) { value = (data->led_state >> i) & 1; break; } return value ? LED_FULL : LED_OFF; } int picolcd_init_leds(struct picolcd_data *data, struct hid_report *report) { struct device *dev = &data->hdev->dev; struct led_classdev *led; size_t name_sz = strlen(dev_name(dev)) + 8; char *name; int i, ret = 0; if (!report) return -ENODEV; if (report->maxfield != 1 || report->field[0]->report_count != 1 || report->field[0]->report_size != 8) { dev_err(dev, "unsupported LED_STATE report"); return -EINVAL; } for (i = 0; i < 8; i++) { led = kzalloc(sizeof(struct led_classdev)+name_sz, GFP_KERNEL); if (!led) { dev_err(dev, "can't allocate memory for LED %d\n", i); ret = -ENOMEM; goto err; } name = (void *)(&led[1]); snprintf(name, name_sz, "%s::GPO%d", dev_name(dev), i); led->name = name; led->brightness = 0; led->max_brightness = 1; led->brightness_get = picolcd_led_get_brightness; led->brightness_set = picolcd_led_set_brightness; data->led[i] = led; ret = led_classdev_register(dev, data->led[i]); if (ret) { data->led[i] = NULL; kfree(led); dev_err(dev, "can't register LED %d\n", i); goto err; } } return 0; err: for (i = 0; i < 8; i++) if (data->led[i]) { led = data->led[i]; data->led[i] = NULL; led_classdev_unregister(led); kfree(led); } return ret; } void picolcd_exit_leds(struct picolcd_data *data) { struct led_classdev *led; int i; for (i = 0; i < 8; i++) { led = data->led[i]; data->led[i] = NULL; if (!led) continue; led_classdev_unregister(led); kfree(led); } } |
| 1 1 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 | // SPDX-License-Identifier: GPL-2.0-or-later /* * * Copyright (C) Alan Cox GW4PTS (alan@lxorguk.ukuu.org.uk) * Copyright (C) Jonathan Naylor G4KLX (g4klx@g4klx.demon.co.uk) * Copyright (C) Joerg Reuter DL1BKE (jreuter@yaina.de) * Copyright (C) Hans-Joachim Hetscher DD8NE (dd8ne@bnv-bamberg.de) */ #include <linux/errno.h> #include <linux/types.h> #include <linux/socket.h> #include <linux/in.h> #include <linux/kernel.h> #include <linux/timer.h> #include <linux/string.h> #include <linux/sockios.h> #include <linux/net.h> #include <linux/slab.h> #include <net/ax25.h> #include <linux/inet.h> #include <linux/netdevice.h> #include <linux/skbuff.h> #include <net/sock.h> #include <net/tcp_states.h> #include <linux/uaccess.h> #include <linux/fcntl.h> #include <linux/mm.h> #include <linux/interrupt.h> /* * Given a fragment, queue it on the fragment queue and if the fragment * is complete, send it back to ax25_rx_iframe. */ static int ax25_rx_fragment(ax25_cb *ax25, struct sk_buff *skb) { struct sk_buff *skbn, *skbo; if (ax25->fragno != 0) { if (!(*skb->data & AX25_SEG_FIRST)) { if ((ax25->fragno - 1) == (*skb->data & AX25_SEG_REM)) { /* Enqueue fragment */ ax25->fragno = *skb->data & AX25_SEG_REM; skb_pull(skb, 1); /* skip fragno */ ax25->fraglen += skb->len; skb_queue_tail(&ax25->frag_queue, skb); /* Last fragment received ? */ if (ax25->fragno == 0) { skbn = alloc_skb(AX25_MAX_HEADER_LEN + ax25->fraglen, GFP_ATOMIC); if (!skbn) { skb_queue_purge(&ax25->frag_queue); return 1; } skb_reserve(skbn, AX25_MAX_HEADER_LEN); skbn->dev = ax25->ax25_dev->dev; skb_reset_network_header(skbn); skb_reset_transport_header(skbn); /* Copy data from the fragments */ while ((skbo = skb_dequeue(&ax25->frag_queue)) != NULL) { skb_copy_from_linear_data(skbo, skb_put(skbn, skbo->len), skbo->len); kfree_skb(skbo); } ax25->fraglen = 0; if (ax25_rx_iframe(ax25, skbn) == 0) kfree_skb(skbn); } return 1; } } } else { /* First fragment received */ if (*skb->data & AX25_SEG_FIRST) { skb_queue_purge(&ax25->frag_queue); ax25->fragno = *skb->data & AX25_SEG_REM; skb_pull(skb, 1); /* skip fragno */ ax25->fraglen = skb->len; skb_queue_tail(&ax25->frag_queue, skb); return 1; } } return 0; } /* * This is where all valid I frames are sent to, to be dispatched to * whichever protocol requires them. */ int ax25_rx_iframe(ax25_cb *ax25, struct sk_buff *skb) { int (*func)(struct sk_buff *, ax25_cb *); unsigned char pid; int queued = 0; if (skb == NULL) return 0; ax25_start_idletimer(ax25); pid = *skb->data; if (pid == AX25_P_IP) { /* working around a TCP bug to keep additional listeners * happy. TCP re-uses the buffer and destroys the original * content. */ struct sk_buff *skbn = skb_copy(skb, GFP_ATOMIC); if (skbn != NULL) { kfree_skb(skb); skb = skbn; } skb_pull(skb, 1); /* Remove PID */ skb->mac_header = skb->network_header; skb_reset_network_header(skb); skb->dev = ax25->ax25_dev->dev; skb->pkt_type = PACKET_HOST; skb->protocol = htons(ETH_P_IP); netif_rx(skb); return 1; } if (pid == AX25_P_SEGMENT) { skb_pull(skb, 1); /* Remove PID */ return ax25_rx_fragment(ax25, skb); } if ((func = ax25_protocol_function(pid)) != NULL) { skb_pull(skb, 1); /* Remove PID */ return (*func)(skb, ax25); } if (ax25->sk != NULL && ax25->ax25_dev->values[AX25_VALUES_CONMODE] == 2) { if ((!ax25->pidincl && ax25->sk->sk_protocol == pid) || ax25->pidincl) { if (sock_queue_rcv_skb(ax25->sk, skb) == 0) queued = 1; else ax25->condition |= AX25_COND_OWN_RX_BUSY; } } return queued; } /* * Higher level upcall for a LAPB frame */ static int ax25_process_rx_frame(ax25_cb *ax25, struct sk_buff *skb, int type, int dama) { int queued = 0; if (ax25->state == AX25_STATE_0) return 0; switch (ax25->ax25_dev->values[AX25_VALUES_PROTOCOL]) { case AX25_PROTO_STD_SIMPLEX: case AX25_PROTO_STD_DUPLEX: queued = ax25_std_frame_in(ax25, skb, type); break; #ifdef CONFIG_AX25_DAMA_SLAVE case AX25_PROTO_DAMA_SLAVE: if (dama || ax25->ax25_dev->dama.slave) queued = ax25_ds_frame_in(ax25, skb, type); else queued = ax25_std_frame_in(ax25, skb, type); break; #endif } return queued; } static int ax25_rcv(struct sk_buff *skb, struct net_device *dev, const ax25_address *dev_addr, struct packet_type *ptype) { ax25_address src, dest, *next_digi = NULL; int type = 0, mine = 0, dama; struct sock *make, *sk; ax25_digi dp, reverse_dp; ax25_cb *ax25; ax25_dev *ax25_dev; /* * Process the AX.25/LAPB frame. */ skb_reset_transport_header(skb); if ((ax25_dev = ax25_dev_ax25dev(dev)) == NULL) goto free; /* * Parse the address header. */ if (ax25_addr_parse(skb->data, skb->len, &src, &dest, &dp, &type, &dama) == NULL) goto free; /* * Ours perhaps ? */ if (dp.lastrepeat + 1 < dp.ndigi) /* Not yet digipeated completely */ next_digi = &dp.calls[dp.lastrepeat + 1]; /* * Pull of the AX.25 headers leaving the CTRL/PID bytes */ skb_pull(skb, ax25_addr_size(&dp)); /* For our port addresses ? */ if (ax25cmp(&dest, dev_addr) == 0 && dp.lastrepeat + 1 == dp.ndigi) mine = 1; /* Also match on any registered callsign from L3/4 */ if (!mine && ax25_listen_mine(&dest, dev) && dp.lastrepeat + 1 == dp.ndigi) mine = 1; /* UI frame - bypass LAPB processing */ if ((*skb->data & ~0x10) == AX25_UI && dp.lastrepeat + 1 == dp.ndigi) { skb_set_transport_header(skb, 2); /* skip control and pid */ ax25_send_to_raw(&dest, skb, skb->data[1]); if (!mine && ax25cmp(&dest, (ax25_address *)dev->broadcast) != 0) goto free; /* Now we are pointing at the pid byte */ switch (skb->data[1]) { case AX25_P_IP: skb_pull(skb,2); /* drop PID/CTRL */ skb_reset_transport_header(skb); skb_reset_network_header(skb); skb->dev = dev; skb->pkt_type = PACKET_HOST; skb->protocol = htons(ETH_P_IP); netif_rx(skb); break; case AX25_P_ARP: skb_pull(skb,2); skb_reset_transport_header(skb); skb_reset_network_header(skb); skb->dev = dev; skb->pkt_type = PACKET_HOST; skb->protocol = htons(ETH_P_ARP); netif_rx(skb); break; case AX25_P_TEXT: /* Now find a suitable dgram socket */ sk = ax25_get_socket(&dest, &src, SOCK_DGRAM); if (sk != NULL) { bh_lock_sock(sk); if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) { kfree_skb(skb); } else { /* * Remove the control and PID. */ skb_pull(skb, 2); if (sock_queue_rcv_skb(sk, skb) != 0) kfree_skb(skb); } bh_unlock_sock(sk); sock_put(sk); } else { kfree_skb(skb); } break; default: kfree_skb(skb); /* Will scan SOCK_AX25 RAW sockets */ break; } return 0; } /* * Is connected mode supported on this device ? * If not, should we DM the incoming frame (except DMs) or * silently ignore them. For now we stay quiet. */ if (ax25_dev->values[AX25_VALUES_CONMODE] == 0) goto free; /* LAPB */ /* AX.25 state 1-4 */ ax25_digi_invert(&dp, &reverse_dp); if ((ax25 = ax25_find_cb(&dest, &src, &reverse_dp, dev)) != NULL) { /* * Process the frame. If it is queued up internally it * returns one otherwise we free it immediately. This * routine itself wakes the user context layers so we do * no further work */ if (ax25_process_rx_frame(ax25, skb, type, dama) == 0) kfree_skb(skb); ax25_cb_put(ax25); return 0; } /* AX.25 state 0 (disconnected) */ /* a) received not a SABM(E) */ if ((*skb->data & ~AX25_PF) != AX25_SABM && (*skb->data & ~AX25_PF) != AX25_SABME) { /* * Never reply to a DM. Also ignore any connects for * addresses that are not our interfaces and not a socket. */ if ((*skb->data & ~AX25_PF) != AX25_DM && mine) ax25_return_dm(dev, &src, &dest, &dp); goto free; } /* b) received SABM(E) */ if (dp.lastrepeat + 1 == dp.ndigi) sk = ax25_find_listener(&dest, 0, dev, SOCK_SEQPACKET); else sk = ax25_find_listener(next_digi, 1, dev, SOCK_SEQPACKET); if (sk != NULL) { bh_lock_sock(sk); if (sk_acceptq_is_full(sk) || (make = ax25_make_new(sk, ax25_dev)) == NULL) { if (mine) ax25_return_dm(dev, &src, &dest, &dp); kfree_skb(skb); bh_unlock_sock(sk); sock_put(sk); return 0; } ax25 = sk_to_ax25(make); skb_set_owner_r(skb, make); skb_queue_head(&sk->sk_receive_queue, skb); make->sk_state = TCP_ESTABLISHED; sk_acceptq_added(sk); bh_unlock_sock(sk); } else { if (!mine) goto free; if ((ax25 = ax25_create_cb()) == NULL) { ax25_return_dm(dev, &src, &dest, &dp); goto free; } ax25_fillin_cb(ax25, ax25_dev); } ax25->source_addr = dest; ax25->dest_addr = src; /* * Sort out any digipeated paths. */ if (dp.ndigi && !ax25->digipeat && (ax25->digipeat = kmalloc(sizeof(ax25_digi), GFP_ATOMIC)) == NULL) { kfree_skb(skb); ax25_destroy_socket(ax25); if (sk) sock_put(sk); return 0; } if (dp.ndigi == 0) { kfree(ax25->digipeat); ax25->digipeat = NULL; } else { /* Reverse the source SABM's path */ memcpy(ax25->digipeat, &reverse_dp, sizeof(ax25_digi)); } if ((*skb->data & ~AX25_PF) == AX25_SABME) { ax25->modulus = AX25_EMODULUS; ax25->window = ax25_dev->values[AX25_VALUES_EWINDOW]; } else { ax25->modulus = AX25_MODULUS; ax25->window = ax25_dev->values[AX25_VALUES_WINDOW]; } ax25_send_control(ax25, AX25_UA, AX25_POLLON, AX25_RESPONSE); #ifdef CONFIG_AX25_DAMA_SLAVE if (dama && ax25->ax25_dev->values[AX25_VALUES_PROTOCOL] == AX25_PROTO_DAMA_SLAVE) ax25_dama_on(ax25); #endif ax25->state = AX25_STATE_3; ax25_cb_add(ax25); ax25_start_heartbeat(ax25); ax25_start_t3timer(ax25); ax25_start_idletimer(ax25); if (sk) { if (!sock_flag(sk, SOCK_DEAD)) sk->sk_data_ready(sk); sock_put(sk); } else { free: kfree_skb(skb); } return 0; } /* * Receive an AX.25 frame via a SLIP interface. */ int ax25_kiss_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *ptype, struct net_device *orig_dev) { skb = skb_share_check(skb, GFP_ATOMIC); if (!skb) return NET_RX_DROP; skb_orphan(skb); if (!net_eq(dev_net(dev), &init_net)) { kfree_skb(skb); return 0; } if ((*skb->data & 0x0F) != 0) { kfree_skb(skb); /* Not a KISS data frame */ return 0; } skb_pull(skb, AX25_KISS_HEADER_LEN); /* Remove the KISS byte */ return ax25_rcv(skb, dev, (const ax25_address *)dev->dev_addr, ptype); } |
| 2 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | // SPDX-License-Identifier: GPL-2.0-only /* * AES-GMAC for IEEE 802.11 BIP-GMAC-128 and BIP-GMAC-256 * Copyright 2015, Qualcomm Atheros, Inc. */ #include <linux/kernel.h> #include <linux/types.h> #include <linux/err.h> #include <crypto/aead.h> #include <crypto/aes.h> #include <net/mac80211.h> #include "key.h" #include "aes_gmac.h" int ieee80211_aes_gmac(struct crypto_aead *tfm, const u8 *aad, u8 *nonce, const u8 *data, size_t data_len, u8 *mic) { struct scatterlist sg[5]; u8 *zero, *__aad, iv[AES_BLOCK_SIZE]; struct aead_request *aead_req; int reqsize = sizeof(*aead_req) + crypto_aead_reqsize(tfm); const __le16 *fc; int ret; if (data_len < GMAC_MIC_LEN) return -EINVAL; aead_req = kzalloc(reqsize + GMAC_MIC_LEN + GMAC_AAD_LEN, GFP_ATOMIC); if (!aead_req) return -ENOMEM; zero = (u8 *)aead_req + reqsize; __aad = zero + GMAC_MIC_LEN; memcpy(__aad, aad, GMAC_AAD_LEN); fc = (const __le16 *)aad; if (ieee80211_is_beacon(*fc)) { /* mask Timestamp field to zero */ sg_init_table(sg, 5); sg_set_buf(&sg[0], __aad, GMAC_AAD_LEN); sg_set_buf(&sg[1], zero, 8); sg_set_buf(&sg[2], data + 8, data_len - 8 - GMAC_MIC_LEN); sg_set_buf(&sg[3], zero, GMAC_MIC_LEN); sg_set_buf(&sg[4], mic, GMAC_MIC_LEN); } else { sg_init_table(sg, 4); sg_set_buf(&sg[0], __aad, GMAC_AAD_LEN); sg_set_buf(&sg[1], data, data_len - GMAC_MIC_LEN); sg_set_buf(&sg[2], zero, GMAC_MIC_LEN); sg_set_buf(&sg[3], mic, GMAC_MIC_LEN); } memcpy(iv, nonce, GMAC_NONCE_LEN); memset(iv + GMAC_NONCE_LEN, 0, sizeof(iv) - GMAC_NONCE_LEN); iv[AES_BLOCK_SIZE - 1] = 0x01; aead_request_set_tfm(aead_req, tfm); aead_request_set_crypt(aead_req, sg, sg, 0, iv); aead_request_set_ad(aead_req, GMAC_AAD_LEN + data_len); ret = crypto_aead_encrypt(aead_req); kfree_sensitive(aead_req); return ret; } struct crypto_aead *ieee80211_aes_gmac_key_setup(const u8 key[], size_t key_len) { struct crypto_aead *tfm; int err; tfm = crypto_alloc_aead("gcm(aes)", 0, CRYPTO_ALG_ASYNC); if (IS_ERR(tfm)) return tfm; err = crypto_aead_setkey(tfm, key, key_len); if (!err) err = crypto_aead_setauthsize(tfm, GMAC_MIC_LEN); if (!err) return tfm; crypto_free_aead(tfm); return ERR_PTR(err); } void ieee80211_aes_gmac_key_free(struct crypto_aead *tfm) { crypto_free_aead(tfm); } |
| 2 34 10 10 43 43 73 51 28 73 73 73 51 28 69 23 55 21 31 20 20 46 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 | // SPDX-License-Identifier: GPL-2.0-or-later /* * fs/inotify_user.c - inotify support for userspace * * Authors: * John McCutchan <ttb@tentacle.dhs.org> * Robert Love <rml@novell.com> * * Copyright (C) 2005 John McCutchan * Copyright 2006 Hewlett-Packard Development Company, L.P. * * Copyright (C) 2009 Eric Paris <Red Hat Inc> * inotify was largely rewritten to make use of the fsnotify infrastructure */ #include <linux/dcache.h> /* d_unlinked */ #include <linux/fs.h> /* struct inode */ #include <linux/fsnotify_backend.h> #include <linux/inotify.h> #include <linux/path.h> /* struct path */ #include <linux/slab.h> /* kmem_* */ #include <linux/types.h> #include <linux/sched.h> #include <linux/sched/user.h> #include <linux/sched/mm.h> #include "inotify.h" /* * Check if 2 events contain the same information. */ static bool event_compare(struct fsnotify_event *old_fsn, struct fsnotify_event *new_fsn) { struct inotify_event_info *old, *new; old = INOTIFY_E(old_fsn); new = INOTIFY_E(new_fsn); if (old->mask & FS_IN_IGNORED) return false; if ((old->mask == new->mask) && (old->wd == new->wd) && (old->name_len == new->name_len) && (!old->name_len || !strcmp(old->name, new->name))) return true; return false; } static int inotify_merge(struct fsnotify_group *group, struct fsnotify_event *event) { struct list_head *list = &group->notification_list; struct fsnotify_event *last_event; last_event = list_entry(list->prev, struct fsnotify_event, list); return event_compare(last_event, event); } int inotify_handle_inode_event(struct fsnotify_mark *inode_mark, u32 mask, struct inode *inode, struct inode *dir, const struct qstr *name, u32 cookie) { struct inotify_inode_mark *i_mark; struct inotify_event_info *event; struct fsnotify_event *fsn_event; struct fsnotify_group *group = inode_mark->group; int ret; int len = 0, wd; int alloc_len = sizeof(struct inotify_event_info); struct mem_cgroup *old_memcg; if (name) { len = name->len; alloc_len += len + 1; } pr_debug("%s: group=%p mark=%p mask=%x\n", __func__, group, inode_mark, mask); i_mark = container_of(inode_mark, struct inotify_inode_mark, fsn_mark); /* * We can be racing with mark being detached. Don't report event with * invalid wd. */ wd = READ_ONCE(i_mark->wd); if (wd == -1) return 0; /* * Whoever is interested in the event, pays for the allocation. Do not * trigger OOM killer in the target monitoring memcg as it may have * security repercussion. */ old_memcg = set_active_memcg(group->memcg); event = kmalloc(alloc_len, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); set_active_memcg(old_memcg); if (unlikely(!event)) { /* * Treat lost event due to ENOMEM the same way as queue * overflow to let userspace know event was lost. */ fsnotify_queue_overflow(group); return -ENOMEM; } /* * We now report FS_ISDIR flag with MOVE_SELF and DELETE_SELF events * for fanotify. inotify never reported IN_ISDIR with those events. * It looks like an oversight, but to avoid the risk of breaking * existing inotify programs, mask the flag out from those events. */ if (mask & (IN_MOVE_SELF | IN_DELETE_SELF)) mask &= ~IN_ISDIR; fsn_event = &event->fse; fsnotify_init_event(fsn_event); event->mask = mask; event->wd = wd; event->sync_cookie = cookie; event->name_len = len; if (len) strscpy(event->name, name->name, event->name_len + 1); ret = fsnotify_add_event(group, fsn_event, inotify_merge); if (ret) { /* Our event wasn't used in the end. Free it. */ fsnotify_destroy_event(group, fsn_event); } if (inode_mark->flags & FSNOTIFY_MARK_FLAG_IN_ONESHOT) fsnotify_destroy_mark(inode_mark, group); return 0; } static void inotify_freeing_mark(struct fsnotify_mark *fsn_mark, struct fsnotify_group *group) { inotify_ignored_and_remove_idr(fsn_mark, group); } /* * This is NEVER supposed to be called. Inotify marks should either have been * removed from the idr when the watch was removed or in the * fsnotify_destroy_mark_by_group() call when the inotify instance was being * torn down. This is only called if the idr is about to be freed but there * are still marks in it. */ static int idr_callback(int id, void *p, void *data) { struct fsnotify_mark *fsn_mark; struct inotify_inode_mark *i_mark; static bool warned = false; if (warned) return 0; warned = true; fsn_mark = p; i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); WARN(1, "inotify closing but id=%d for fsn_mark=%p in group=%p still in " "idr. Probably leaking memory\n", id, p, data); /* * I'm taking the liberty of assuming that the mark in question is a * valid address and I'm dereferencing it. This might help to figure * out why we got here and the panic is no worse than the original * BUG() that was here. */ if (fsn_mark) printk(KERN_WARNING "fsn_mark->group=%p wd=%d\n", fsn_mark->group, i_mark->wd); return 0; } static void inotify_free_group_priv(struct fsnotify_group *group) { /* ideally the idr is empty and we won't hit the BUG in the callback */ idr_for_each(&group->inotify_data.idr, idr_callback, group); idr_destroy(&group->inotify_data.idr); if (group->inotify_data.ucounts) dec_inotify_instances(group->inotify_data.ucounts); } static void inotify_free_event(struct fsnotify_group *group, struct fsnotify_event *fsn_event) { kfree(INOTIFY_E(fsn_event)); } /* ding dong the mark is dead */ static void inotify_free_mark(struct fsnotify_mark *fsn_mark) { struct inotify_inode_mark *i_mark; i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); kmem_cache_free(inotify_inode_mark_cachep, i_mark); } const struct fsnotify_ops inotify_fsnotify_ops = { .handle_inode_event = inotify_handle_inode_event, .free_group_priv = inotify_free_group_priv, .free_event = inotify_free_event, .freeing_mark = inotify_freeing_mark, .free_mark = inotify_free_mark, }; |
| 2 2 13 11 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 | // SPDX-License-Identifier: GPL-2.0-or-later #include <linux/module.h> #include <linux/netdevice.h> #include <linux/mii.h> #include <linux/usb.h> #include <linux/usb/cdc.h> #include <linux/usb/usbnet.h> #include <linux/usb/r8152.h> #define OCP_BASE 0xe86c static int pla_read_word(struct usbnet *dev, u16 index) { u16 byen = BYTE_EN_WORD; u8 shift = index & 2; __le32 tmp; int ret; if (shift) byen <<= shift; index &= ~3; ret = usbnet_read_cmd(dev, RTL8152_REQ_GET_REGS, RTL8152_REQT_READ, index, MCU_TYPE_PLA | byen, &tmp, sizeof(tmp)); if (ret < 0) goto out; ret = __le32_to_cpu(tmp); ret >>= (shift * 8); ret &= 0xffff; out: return ret; } static int pla_write_word(struct usbnet *dev, u16 index, u32 data) { u32 mask = 0xffff; u16 byen = BYTE_EN_WORD; u8 shift = index & 2; __le32 tmp; int ret; data &= mask; if (shift) { byen <<= shift; mask <<= (shift * 8); data <<= (shift * 8); } index &= ~3; ret = usbnet_read_cmd(dev, RTL8152_REQ_GET_REGS, RTL8152_REQT_READ, index, MCU_TYPE_PLA | byen, &tmp, sizeof(tmp)); if (ret < 0) goto out; data |= __le32_to_cpu(tmp) & ~mask; tmp = __cpu_to_le32(data); ret = usbnet_write_cmd(dev, RTL8152_REQ_SET_REGS, RTL8152_REQT_WRITE, index, MCU_TYPE_PLA | byen, &tmp, sizeof(tmp)); out: return ret; } static int r8153_ecm_mdio_read(struct net_device *netdev, int phy_id, int reg) { struct usbnet *dev = netdev_priv(netdev); int ret; ret = pla_write_word(dev, OCP_BASE, 0xa000); if (ret < 0) goto out; ret = pla_read_word(dev, 0xb400 + reg * 2); out: return ret; } static void r8153_ecm_mdio_write(struct net_device *netdev, int phy_id, int reg, int val) { struct usbnet *dev = netdev_priv(netdev); int ret; ret = pla_write_word(dev, OCP_BASE, 0xa000); if (ret < 0) return; ret = pla_write_word(dev, 0xb400 + reg * 2, val); } static int r8153_bind(struct usbnet *dev, struct usb_interface *intf) { int status; status = usbnet_cdc_bind(dev, intf); if (status < 0) return status; dev->mii.dev = dev->net; dev->mii.mdio_read = r8153_ecm_mdio_read; dev->mii.mdio_write = r8153_ecm_mdio_write; dev->mii.reg_num_mask = 0x1f; dev->mii.supports_gmii = 1; return status; } static const struct driver_info r8153_info = { .description = "RTL8153 ECM Device", .flags = FLAG_ETHER, .bind = r8153_bind, .unbind = usbnet_cdc_unbind, .status = usbnet_cdc_status, .manage_power = usbnet_manage_power, }; static const struct usb_device_id products[] = { /* Realtek RTL8153 Based USB 3.0 Ethernet Adapters */ { USB_DEVICE_AND_INTERFACE_INFO(VENDOR_ID_REALTEK, 0x8153, USB_CLASS_COMM, USB_CDC_SUBCLASS_ETHERNET, USB_CDC_PROTO_NONE), .driver_info = (unsigned long)&r8153_info, }, /* Lenovo Powered USB-C Travel Hub (4X90S92381, based on Realtek RTL8153) */ { USB_DEVICE_AND_INTERFACE_INFO(VENDOR_ID_LENOVO, 0x721e, USB_CLASS_COMM, USB_CDC_SUBCLASS_ETHERNET, USB_CDC_PROTO_NONE), .driver_info = (unsigned long)&r8153_info, }, /* Lenovo ThinkPad Hybrid USB-C with USB-A Dock (40af0135eu, based on Realtek RTL8153) */ { USB_DEVICE_AND_INTERFACE_INFO(VENDOR_ID_LENOVO, 0xa359, USB_CLASS_COMM, USB_CDC_SUBCLASS_ETHERNET, USB_CDC_PROTO_NONE), .driver_info = (unsigned long)&r8153_info, }, { }, /* END */ }; MODULE_DEVICE_TABLE(usb, products); static int rtl8153_ecm_probe(struct usb_interface *intf, const struct usb_device_id *id) { #if IS_REACHABLE(CONFIG_USB_RTL8152) if (rtl8152_get_version(intf)) return -ENODEV; #endif return usbnet_probe(intf, id); } static struct usb_driver r8153_ecm_driver = { .name = "r8153_ecm", .id_table = products, .probe = rtl8153_ecm_probe, .disconnect = usbnet_disconnect, .suspend = usbnet_suspend, .resume = usbnet_resume, .reset_resume = usbnet_resume, .supports_autosuspend = 1, .disable_hub_initiated_lpm = 1, }; module_usb_driver(r8153_ecm_driver); MODULE_AUTHOR("Hayes Wang"); MODULE_DESCRIPTION("Realtek USB ECM device"); MODULE_LICENSE("GPL"); |
| 101 38 14 1400 1394 15 20 102 16 149 1539 129 1 124 123 123 1371 4 14 19 33 31 7 30 25 8 1396 26 94 2 11 26 2 40 86 145 1546 6 2 1 2 4 17 13 7 2 1402 1 1 5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _LINUX_VIRTIO_NET_H #define _LINUX_VIRTIO_NET_H #include <linux/if_vlan.h> #include <linux/ip.h> #include <linux/ipv6.h> #include <linux/udp.h> #include <uapi/linux/tcp.h> #include <uapi/linux/virtio_net.h> static inline bool virtio_net_hdr_match_proto(__be16 protocol, __u8 gso_type) { switch (gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { case VIRTIO_NET_HDR_GSO_TCPV4: return protocol == cpu_to_be16(ETH_P_IP); case VIRTIO_NET_HDR_GSO_TCPV6: return protocol == cpu_to_be16(ETH_P_IPV6); case VIRTIO_NET_HDR_GSO_UDP: case VIRTIO_NET_HDR_GSO_UDP_L4: return protocol == cpu_to_be16(ETH_P_IP) || protocol == cpu_to_be16(ETH_P_IPV6); default: return false; } } static inline int virtio_net_hdr_set_proto(struct sk_buff *skb, const struct virtio_net_hdr *hdr) { if (skb->protocol) return 0; switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { case VIRTIO_NET_HDR_GSO_TCPV4: case VIRTIO_NET_HDR_GSO_UDP: case VIRTIO_NET_HDR_GSO_UDP_L4: skb->protocol = cpu_to_be16(ETH_P_IP); break; case VIRTIO_NET_HDR_GSO_TCPV6: skb->protocol = cpu_to_be16(ETH_P_IPV6); break; default: return -EINVAL; } return 0; } static inline int __virtio_net_hdr_to_skb(struct sk_buff *skb, const struct virtio_net_hdr *hdr, bool little_endian, u8 hdr_gso_type) { unsigned int nh_min_len = sizeof(struct iphdr); unsigned int gso_type = 0; unsigned int thlen = 0; unsigned int p_off = 0; unsigned int ip_proto; if (hdr_gso_type != VIRTIO_NET_HDR_GSO_NONE) { switch (hdr_gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { case VIRTIO_NET_HDR_GSO_TCPV4: gso_type = SKB_GSO_TCPV4; ip_proto = IPPROTO_TCP; thlen = sizeof(struct tcphdr); break; case VIRTIO_NET_HDR_GSO_TCPV6: gso_type = SKB_GSO_TCPV6; ip_proto = IPPROTO_TCP; thlen = sizeof(struct tcphdr); nh_min_len = sizeof(struct ipv6hdr); break; case VIRTIO_NET_HDR_GSO_UDP: gso_type = SKB_GSO_UDP; ip_proto = IPPROTO_UDP; thlen = sizeof(struct udphdr); break; case VIRTIO_NET_HDR_GSO_UDP_L4: gso_type = SKB_GSO_UDP_L4; ip_proto = IPPROTO_UDP; thlen = sizeof(struct udphdr); break; default: return -EINVAL; } if (hdr_gso_type & VIRTIO_NET_HDR_GSO_ECN) gso_type |= SKB_GSO_TCP_ECN; if (hdr->gso_size == 0) return -EINVAL; } skb_reset_mac_header(skb); if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { u32 start = __virtio16_to_cpu(little_endian, hdr->csum_start); u32 off = __virtio16_to_cpu(little_endian, hdr->csum_offset); u32 needed = start + max_t(u32, thlen, off + sizeof(__sum16)); if (!pskb_may_pull(skb, needed)) return -EINVAL; if (!skb_partial_csum_set(skb, start, off)) return -EINVAL; if (skb_transport_offset(skb) < nh_min_len) return -EINVAL; nh_min_len = skb_transport_offset(skb); p_off = nh_min_len + thlen; if (!pskb_may_pull(skb, p_off)) return -EINVAL; } else { /* gso packets without NEEDS_CSUM do not set transport_offset. * probe and drop if does not match one of the above types. */ if (gso_type && skb->network_header) { struct flow_keys_basic keys; if (!skb->protocol) { __be16 protocol = dev_parse_header_protocol(skb); if (!protocol) virtio_net_hdr_set_proto(skb, hdr); else if (!virtio_net_hdr_match_proto(protocol, hdr_gso_type)) return -EINVAL; else skb->protocol = protocol; } retry: if (!skb_flow_dissect_flow_keys_basic(NULL, skb, &keys, NULL, 0, 0, 0, 0)) { /* UFO does not specify ipv4 or 6: try both */ if (gso_type & SKB_GSO_UDP && skb->protocol == htons(ETH_P_IP)) { skb->protocol = htons(ETH_P_IPV6); goto retry; } return -EINVAL; } p_off = keys.control.thoff + thlen; if (!pskb_may_pull(skb, p_off) || keys.basic.ip_proto != ip_proto) return -EINVAL; skb_set_transport_header(skb, keys.control.thoff); } else if (gso_type) { p_off = nh_min_len + thlen; if (!pskb_may_pull(skb, p_off)) return -EINVAL; } } if (hdr_gso_type != VIRTIO_NET_HDR_GSO_NONE) { u16 gso_size = __virtio16_to_cpu(little_endian, hdr->gso_size); unsigned int nh_off = p_off; struct skb_shared_info *shinfo = skb_shinfo(skb); switch (gso_type & ~SKB_GSO_TCP_ECN) { case SKB_GSO_UDP: /* UFO may not include transport header in gso_size. */ nh_off -= thlen; break; case SKB_GSO_UDP_L4: if (!(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) return -EINVAL; if (skb->csum_offset != offsetof(struct udphdr, check)) return -EINVAL; if (skb->len - p_off > gso_size * UDP_MAX_SEGMENTS) return -EINVAL; if (gso_type != SKB_GSO_UDP_L4) return -EINVAL; break; case SKB_GSO_TCPV4: case SKB_GSO_TCPV6: if (skb->ip_summed == CHECKSUM_PARTIAL && skb->csum_offset != offsetof(struct tcphdr, check)) return -EINVAL; break; } /* Kernel has a special handling for GSO_BY_FRAGS. */ if (gso_size == GSO_BY_FRAGS) return -EINVAL; /* Too small packets are not really GSO ones. */ if (skb->len - nh_off > gso_size) { shinfo->gso_size = gso_size; shinfo->gso_type = gso_type; /* Header must be checked, and gso_segs computed. */ shinfo->gso_type |= SKB_GSO_DODGY; shinfo->gso_segs = 0; } } return 0; } static inline int virtio_net_hdr_to_skb(struct sk_buff *skb, const struct virtio_net_hdr *hdr, bool little_endian) { return __virtio_net_hdr_to_skb(skb, hdr, little_endian, hdr->gso_type); } static inline int virtio_net_hdr_from_skb(const struct sk_buff *skb, struct virtio_net_hdr *hdr, bool little_endian, bool has_data_valid, int vlan_hlen) { memset(hdr, 0, sizeof(*hdr)); /* no info leak */ if (skb_is_gso(skb)) { struct skb_shared_info *sinfo = skb_shinfo(skb); /* This is a hint as to how much should be linear. */ hdr->hdr_len = __cpu_to_virtio16(little_endian, skb_headlen(skb)); hdr->gso_size = __cpu_to_virtio16(little_endian, sinfo->gso_size); if (sinfo->gso_type & SKB_GSO_TCPV4) hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; else if (sinfo->gso_type & SKB_GSO_TCPV6) hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; else if (sinfo->gso_type & SKB_GSO_UDP_L4) hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP_L4; else return -EINVAL; if (sinfo->gso_type & SKB_GSO_TCP_ECN) hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; } else hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE; if (skb->ip_summed == CHECKSUM_PARTIAL) { hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; hdr->csum_start = __cpu_to_virtio16(little_endian, skb_checksum_start_offset(skb) + vlan_hlen); hdr->csum_offset = __cpu_to_virtio16(little_endian, skb->csum_offset); } else if (has_data_valid && skb->ip_summed == CHECKSUM_UNNECESSARY) { hdr->flags = VIRTIO_NET_HDR_F_DATA_VALID; } /* else everything is zero */ return 0; } static inline unsigned int virtio_l3min(bool is_ipv6) { return is_ipv6 ? sizeof(struct ipv6hdr) : sizeof(struct iphdr); } static inline int virtio_net_hdr_tnl_to_skb(struct sk_buff *skb, const struct virtio_net_hdr_v1_hash_tunnel *vhdr, bool tnl_hdr_negotiated, bool tnl_csum_negotiated, bool little_endian) { const struct virtio_net_hdr *hdr = (const struct virtio_net_hdr *)vhdr; unsigned int inner_nh, outer_th, inner_th; unsigned int inner_l3min, outer_l3min; u8 gso_inner_type, gso_tunnel_type; bool outer_isv6, inner_isv6; int ret; gso_tunnel_type = hdr->gso_type & VIRTIO_NET_HDR_GSO_UDP_TUNNEL; if (!gso_tunnel_type) return virtio_net_hdr_to_skb(skb, hdr, little_endian); /* Tunnel not supported/negotiated, but the hdr asks for it. */ if (!tnl_hdr_negotiated) return -EINVAL; /* Either ipv4 or ipv6. */ if (gso_tunnel_type == VIRTIO_NET_HDR_GSO_UDP_TUNNEL) return -EINVAL; /* The UDP tunnel must carry a GSO packet, but no UFO. */ gso_inner_type = hdr->gso_type & ~(VIRTIO_NET_HDR_GSO_ECN | VIRTIO_NET_HDR_GSO_UDP_TUNNEL); if (!gso_inner_type || gso_inner_type == VIRTIO_NET_HDR_GSO_UDP) return -EINVAL; /* Rely on csum being present. */ if (!(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) return -EINVAL; /* Validate offsets. */ outer_isv6 = gso_tunnel_type & VIRTIO_NET_HDR_GSO_UDP_TUNNEL_IPV6; inner_isv6 = gso_inner_type == VIRTIO_NET_HDR_GSO_TCPV6; inner_l3min = virtio_l3min(inner_isv6); outer_l3min = ETH_HLEN + virtio_l3min(outer_isv6); inner_th = __virtio16_to_cpu(little_endian, hdr->csum_start); inner_nh = le16_to_cpu(vhdr->inner_nh_offset); outer_th = le16_to_cpu(vhdr->outer_th_offset); if (outer_th < outer_l3min || inner_nh < outer_th + sizeof(struct udphdr) || inner_th < inner_nh + inner_l3min) return -EINVAL; /* Let the basic parsing deal with plain GSO features. */ ret = __virtio_net_hdr_to_skb(skb, hdr, true, hdr->gso_type & ~gso_tunnel_type); if (ret) return ret; /* In case of USO, the inner protocol is still unknown and * `inner_isv6` is just a guess, additional parsing is needed. * The previous validation ensures that accessing an ipv4 inner * network header is safe. */ if (gso_inner_type == VIRTIO_NET_HDR_GSO_UDP_L4) { struct iphdr *iphdr = (struct iphdr *)(skb->data + inner_nh); inner_isv6 = iphdr->version == 6; inner_l3min = virtio_l3min(inner_isv6); if (inner_th < inner_nh + inner_l3min) return -EINVAL; } skb_set_inner_protocol(skb, inner_isv6 ? htons(ETH_P_IPV6) : htons(ETH_P_IP)); if (hdr->flags & VIRTIO_NET_HDR_F_UDP_TUNNEL_CSUM) { if (!tnl_csum_negotiated) return -EINVAL; skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL_CSUM; } else { skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_TUNNEL; } skb->inner_transport_header = inner_th + skb_headroom(skb); skb->inner_network_header = inner_nh + skb_headroom(skb); skb->inner_mac_header = inner_nh + skb_headroom(skb); skb->transport_header = outer_th + skb_headroom(skb); skb->encapsulation = 1; return 0; } /* Checksum-related fields validation for the driver */ static inline int virtio_net_handle_csum_offload(struct sk_buff *skb, struct virtio_net_hdr *hdr, bool tnl_csum_negotiated) { if (!(hdr->gso_type & VIRTIO_NET_HDR_GSO_UDP_TUNNEL)) { if (!(hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID)) return 0; skb->ip_summed = CHECKSUM_UNNECESSARY; if (!(hdr->flags & VIRTIO_NET_HDR_F_UDP_TUNNEL_CSUM)) return 0; /* tunnel csum packets are invalid when the related * feature has not been negotiated */ if (!tnl_csum_negotiated) return -EINVAL; skb->csum_level = 1; return 0; } /* DATA_VALID is mutually exclusive with NEEDS_CSUM, and GSO * over UDP tunnel requires the latter */ if (hdr->flags & VIRTIO_NET_HDR_F_DATA_VALID) return -EINVAL; return 0; } /* * vlan_hlen always refers to the outermost MAC header. That also * means it refers to the only MAC header, if the packet does not carry * any encapsulation. */ static inline int virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, struct virtio_net_hdr_v1_hash_tunnel *vhdr, bool tnl_hdr_negotiated, bool little_endian, int vlan_hlen) { struct virtio_net_hdr *hdr = (struct virtio_net_hdr *)vhdr; unsigned int inner_nh, outer_th; int tnl_gso_type; int ret; tnl_gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM); if (!tnl_gso_type) return virtio_net_hdr_from_skb(skb, hdr, little_endian, false, vlan_hlen); /* Tunnel support not negotiated but skb ask for it. */ if (!tnl_hdr_negotiated) return -EINVAL; vhdr->hash_hdr.hash_value_lo = 0; vhdr->hash_hdr.hash_value_hi = 0; vhdr->hash_hdr.hash_report = 0; vhdr->hash_hdr.padding = 0; /* Let the basic parsing deal with plain GSO features. */ skb_shinfo(skb)->gso_type &= ~tnl_gso_type; ret = virtio_net_hdr_from_skb(skb, hdr, true, false, vlan_hlen); skb_shinfo(skb)->gso_type |= tnl_gso_type; if (ret) return ret; if (skb->protocol == htons(ETH_P_IPV6)) hdr->gso_type |= VIRTIO_NET_HDR_GSO_UDP_TUNNEL_IPV6; else hdr->gso_type |= VIRTIO_NET_HDR_GSO_UDP_TUNNEL_IPV4; if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM) hdr->flags |= VIRTIO_NET_HDR_F_UDP_TUNNEL_CSUM; inner_nh = skb->inner_network_header - skb_headroom(skb); outer_th = skb->transport_header - skb_headroom(skb); vhdr->inner_nh_offset = cpu_to_le16(inner_nh); vhdr->outer_th_offset = cpu_to_le16(outer_th); return 0; } #endif /* _LINUX_VIRTIO_NET_H */ |
| 157 157 157 157 156 155 155 155 154 154 119 119 119 119 16 78 7 57 57 1 57 74 74 97 4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _MM_SWAP_H #define _MM_SWAP_H #include <linux/atomic.h> /* for atomic_long_t */ struct mempolicy; struct swap_iocb; extern int page_cluster; #ifdef CONFIG_THP_SWAP #define SWAPFILE_CLUSTER HPAGE_PMD_NR #define swap_entry_order(order) (order) #else #define SWAPFILE_CLUSTER 256 #define swap_entry_order(order) 0 #endif extern struct swap_info_struct *swap_info[]; /* * We use this to track usage of a cluster. A cluster is a block of swap disk * space with SWAPFILE_CLUSTER pages long and naturally aligns in disk. All * free clusters are organized into a list. We fetch an entry from the list to * get a free cluster. * * The flags field determines if a cluster is free. This is * protected by cluster lock. */ struct swap_cluster_info { spinlock_t lock; /* * Protect swap_cluster_info fields * other than list, and swap_info_struct->swap_map * elements corresponding to the swap cluster. */ u16 count; u8 flags; u8 order; atomic_long_t __rcu *table; /* Swap table entries, see mm/swap_table.h */ struct list_head list; }; /* All on-list cluster must have a non-zero flag. */ enum swap_cluster_flags { CLUSTER_FLAG_NONE = 0, /* For temporary off-list cluster */ CLUSTER_FLAG_FREE, CLUSTER_FLAG_NONFULL, CLUSTER_FLAG_FRAG, /* Clusters with flags above are allocatable */ CLUSTER_FLAG_USABLE = CLUSTER_FLAG_FRAG, CLUSTER_FLAG_FULL, CLUSTER_FLAG_DISCARD, CLUSTER_FLAG_MAX, }; #ifdef CONFIG_SWAP #include <linux/swapops.h> /* for swp_offset */ #include <linux/blk_types.h> /* for bio_end_io_t */ static inline unsigned int swp_cluster_offset(swp_entry_t entry) { return swp_offset(entry) % SWAPFILE_CLUSTER; } /* * Callers of all helpers below must ensure the entry, type, or offset is * valid, and protect the swap device with reference count or locks. */ static inline struct swap_info_struct *__swap_type_to_info(int type) { struct swap_info_struct *si; si = READ_ONCE(swap_info[type]); /* rcu_dereference() */ VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */ return si; } static inline struct swap_info_struct *__swap_entry_to_info(swp_entry_t entry) { return __swap_type_to_info(swp_type(entry)); } static inline struct swap_cluster_info *__swap_offset_to_cluster( struct swap_info_struct *si, pgoff_t offset) { VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */ VM_WARN_ON_ONCE(offset >= si->max); return &si->cluster_info[offset / SWAPFILE_CLUSTER]; } static inline struct swap_cluster_info *__swap_entry_to_cluster(swp_entry_t entry) { return __swap_offset_to_cluster(__swap_entry_to_info(entry), swp_offset(entry)); } static __always_inline struct swap_cluster_info *__swap_cluster_lock( struct swap_info_struct *si, unsigned long offset, bool irq) { struct swap_cluster_info *ci = __swap_offset_to_cluster(si, offset); /* * Nothing modifies swap cache in an IRQ context. All access to * swap cache is wrapped by swap_cache_* helpers, and swap cache * writeback is handled outside of IRQs. Swapin or swapout never * occurs in IRQ, and neither does in-place split or replace. * * Besides, modifying swap cache requires synchronization with * swap_map, which was never IRQ safe. */ VM_WARN_ON_ONCE(!in_task()); VM_WARN_ON_ONCE(percpu_ref_is_zero(&si->users)); /* race with swapoff */ if (irq) spin_lock_irq(&ci->lock); else spin_lock(&ci->lock); return ci; } /** * swap_cluster_lock - Lock and return the swap cluster of given offset. * @si: swap device the cluster belongs to. * @offset: the swap entry offset, pointing to a valid slot. * * Context: The caller must ensure the offset is in the valid range and * protect the swap device with reference count or locks. */ static inline struct swap_cluster_info *swap_cluster_lock( struct swap_info_struct *si, unsigned long offset) { return __swap_cluster_lock(si, offset, false); } static inline struct swap_cluster_info *__swap_cluster_get_and_lock( const struct folio *folio, bool irq) { VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio); return __swap_cluster_lock(__swap_entry_to_info(folio->swap), swp_offset(folio->swap), irq); } /* * swap_cluster_get_and_lock - Locks the cluster that holds a folio's entries. * @folio: The folio. * * This locks and returns the swap cluster that contains a folio's swap * entries. The swap entries of a folio are always in one single cluster. * The folio has to be locked so its swap entries won't change and the * cluster won't be freed. * * Context: Caller must ensure the folio is locked and in the swap cache. * Return: Pointer to the swap cluster. */ static inline struct swap_cluster_info *swap_cluster_get_and_lock( const struct folio *folio) { return __swap_cluster_get_and_lock(folio, false); } /* * swap_cluster_get_and_lock_irq - Locks the cluster that holds a folio's entries. * @folio: The folio. * * Same as swap_cluster_get_and_lock but also disable IRQ. * * Context: Caller must ensure the folio is locked and in the swap cache. * Return: Pointer to the swap cluster. */ static inline struct swap_cluster_info *swap_cluster_get_and_lock_irq( const struct folio *folio) { return __swap_cluster_get_and_lock(folio, true); } static inline void swap_cluster_unlock(struct swap_cluster_info *ci) { spin_unlock(&ci->lock); } static inline void swap_cluster_unlock_irq(struct swap_cluster_info *ci) { spin_unlock_irq(&ci->lock); } /* linux/mm/page_io.c */ int sio_pool_init(void); struct swap_iocb; void swap_read_folio(struct folio *folio, struct swap_iocb **plug); void __swap_read_unplug(struct swap_iocb *plug); static inline void swap_read_unplug(struct swap_iocb *plug) { if (unlikely(plug)) __swap_read_unplug(plug); } void swap_write_unplug(struct swap_iocb *sio); int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug); void __swap_writepage(struct folio *folio, struct swap_iocb **swap_plug); /* linux/mm/swap_state.c */ extern struct address_space swap_space __ro_after_init; static inline struct address_space *swap_address_space(swp_entry_t entry) { return &swap_space; } /* * Return the swap device position of the swap entry. */ static inline loff_t swap_dev_pos(swp_entry_t entry) { return ((loff_t)swp_offset(entry)) << PAGE_SHIFT; } /** * folio_matches_swap_entry - Check if a folio matches a given swap entry. * @folio: The folio. * @entry: The swap entry to check against. * * Context: The caller should have the folio locked to ensure it's stable * and nothing will move it in or out of the swap cache. * Return: true or false. */ static inline bool folio_matches_swap_entry(const struct folio *folio, swp_entry_t entry) { swp_entry_t folio_entry = folio->swap; long nr_pages = folio_nr_pages(folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); if (!folio_test_swapcache(folio)) return false; VM_WARN_ON_ONCE_FOLIO(!IS_ALIGNED(folio_entry.val, nr_pages), folio); return folio_entry.val == round_down(entry.val, nr_pages); } /* * All swap cache helpers below require the caller to ensure the swap entries * used are valid and stablize the device by any of the following ways: * - Hold a reference by get_swap_device(): this ensures a single entry is * valid and increases the swap device's refcount. * - Locking a folio in the swap cache: this ensures the folio's swap entries * are valid and pinned, also implies reference to the device. * - Locking anything referencing the swap entry: e.g. PTL that protects * swap entries in the page table, similar to locking swap cache folio. * - See the comment of get_swap_device() for more complex usage. */ struct folio *swap_cache_get_folio(swp_entry_t entry); void *swap_cache_get_shadow(swp_entry_t entry); void swap_cache_add_folio(struct folio *folio, swp_entry_t entry, void **shadow); void swap_cache_del_folio(struct folio *folio); /* Below helpers require the caller to lock and pass in the swap cluster. */ void __swap_cache_del_folio(struct swap_cluster_info *ci, struct folio *folio, swp_entry_t entry, void *shadow); void __swap_cache_replace_folio(struct swap_cluster_info *ci, struct folio *old, struct folio *new); void __swap_cache_clear_shadow(swp_entry_t entry, int nr_ents); void show_swap_cache_info(void); void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr); struct folio *read_swap_cache_async(swp_entry_t entry, gfp_t gfp_mask, struct vm_area_struct *vma, unsigned long addr, struct swap_iocb **plug); struct folio *__read_swap_cache_async(swp_entry_t entry, gfp_t gfp_flags, struct mempolicy *mpol, pgoff_t ilx, bool *new_page_allocated, bool skip_if_exists); struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t flag, struct mempolicy *mpol, pgoff_t ilx); struct folio *swapin_readahead(swp_entry_t entry, gfp_t flag, struct vm_fault *vmf); void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, unsigned long addr); static inline unsigned int folio_swap_flags(struct folio *folio) { return __swap_entry_to_info(folio->swap)->flags; } /* * Return the count of contiguous swap entries that share the same * zeromap status as the starting entry. If is_zeromap is not NULL, * it will return the zeromap status of the starting entry. */ static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr, bool *is_zeromap) { struct swap_info_struct *sis = __swap_entry_to_info(entry); unsigned long start = swp_offset(entry); unsigned long end = start + max_nr; bool first_bit; first_bit = test_bit(start, sis->zeromap); if (is_zeromap) *is_zeromap = first_bit; if (max_nr <= 1) return max_nr; if (first_bit) return find_next_zero_bit(sis->zeromap, end, start) - start; else return find_next_bit(sis->zeromap, end, start) - start; } static inline int non_swapcache_batch(swp_entry_t entry, int max_nr) { struct swap_info_struct *si = __swap_entry_to_info(entry); pgoff_t offset = swp_offset(entry); int i; /* * While allocating a large folio and doing mTHP swapin, we need to * ensure all entries are not cached, otherwise, the mTHP folio will * be in conflict with the folio in swap cache. */ for (i = 0; i < max_nr; i++) { if ((si->swap_map[offset + i] & SWAP_HAS_CACHE)) return i; } return i; } #else /* CONFIG_SWAP */ struct swap_iocb; static inline struct swap_cluster_info *swap_cluster_lock( struct swap_info_struct *si, pgoff_t offset, bool irq) { return NULL; } static inline struct swap_cluster_info *swap_cluster_get_and_lock( struct folio *folio) { return NULL; } static inline struct swap_cluster_info *swap_cluster_get_and_lock_irq( struct folio *folio) { return NULL; } static inline void swap_cluster_unlock(struct swap_cluster_info *ci) { } static inline void swap_cluster_unlock_irq(struct swap_cluster_info *ci) { } static inline struct swap_info_struct *__swap_entry_to_info(swp_entry_t entry) { return NULL; } static inline void swap_read_folio(struct folio *folio, struct swap_iocb **plug) { } static inline void swap_write_unplug(struct swap_iocb *sio) { } static inline struct address_space *swap_address_space(swp_entry_t entry) { return NULL; } static inline bool folio_matches_swap_entry(const struct folio *folio, swp_entry_t entry) { return false; } static inline void show_swap_cache_info(void) { } static inline struct folio *swap_cluster_readahead(swp_entry_t entry, gfp_t gfp_mask, struct mempolicy *mpol, pgoff_t ilx) { return NULL; } static inline struct folio *swapin_readahead(swp_entry_t swp, gfp_t gfp_mask, struct vm_fault *vmf) { return NULL; } static inline void swap_update_readahead(struct folio *folio, struct vm_area_struct *vma, unsigned long addr) { } static inline int swap_writeout(struct folio *folio, struct swap_iocb **swap_plug) { return 0; } static inline void swapcache_clear(struct swap_info_struct *si, swp_entry_t entry, int nr) { } static inline struct folio *swap_cache_get_folio(swp_entry_t entry) { return NULL; } static inline void *swap_cache_get_shadow(swp_entry_t entry) { return NULL; } static inline void swap_cache_add_folio(struct folio *folio, swp_entry_t entry, void **shadow) { } static inline void swap_cache_del_folio(struct folio *folio) { } static inline void __swap_cache_del_folio(struct swap_cluster_info *ci, struct folio *folio, swp_entry_t entry, void *shadow) { } static inline void __swap_cache_replace_folio(struct swap_cluster_info *ci, struct folio *old, struct folio *new) { } static inline unsigned int folio_swap_flags(struct folio *folio) { return 0; } static inline int swap_zeromap_batch(swp_entry_t entry, int max_nr, bool *has_zeromap) { return 0; } static inline int non_swapcache_batch(swp_entry_t entry, int max_nr) { return 0; } #endif /* CONFIG_SWAP */ /** * folio_index - File index of a folio. * @folio: The folio. * * For a folio which is either in the page cache or the swap cache, * return its index within the address_space it belongs to. If you know * the folio is definitely in the page cache, you can look at the folio's * index directly. * * Return: The index (offset in units of pages) of a folio in its file. */ static inline pgoff_t folio_index(struct folio *folio) { #ifdef CONFIG_SWAP if (unlikely(folio_test_swapcache(folio))) return swp_offset(folio->swap); #endif return folio->index; } #endif /* _MM_SWAP_H */ |
| 14 10 8 14 14 14 6 6 10 5 5 5 2 3 5 2 3 2 2 9 2 8 8 4 4 8 7 4 3 3 2 3 2 1 7 2 2 4 7 8 4 4 3 7 4 4 7 7 7 4 4 4 9 7 2 7 2 1 5 7 2 5 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 | // SPDX-License-Identifier: GPL-2.0 /* * RTC subsystem, interface functions * * Copyright (C) 2005 Tower Technologies * Author: Alessandro Zummo <a.zummo@towertech.it> * * based on arch/arm/common/rtctime.c */ #include <linux/rtc.h> #include <linux/sched.h> #include <linux/module.h> #include <linux/log2.h> #include <linux/workqueue.h> #define CREATE_TRACE_POINTS #include <trace/events/rtc.h> static int rtc_timer_enqueue(struct rtc_device *rtc, struct rtc_timer *timer); static void rtc_timer_remove(struct rtc_device *rtc, struct rtc_timer *timer); static void rtc_add_offset(struct rtc_device *rtc, struct rtc_time *tm) { time64_t secs; if (!rtc->offset_secs) return; secs = rtc_tm_to_time64(tm); /* * Since the reading time values from RTC device are always in the RTC * original valid range, but we need to skip the overlapped region * between expanded range and original range, which is no need to add * the offset. */ if ((rtc->start_secs > rtc->range_min && secs >= rtc->start_secs) || (rtc->start_secs < rtc->range_min && secs <= (rtc->start_secs + rtc->range_max - rtc->range_min))) return; rtc_time64_to_tm(secs + rtc->offset_secs, tm); } static void rtc_subtract_offset(struct rtc_device *rtc, struct rtc_time *tm) { time64_t secs; if (!rtc->offset_secs) return; secs = rtc_tm_to_time64(tm); /* * If the setting time values are in the valid range of RTC hardware * device, then no need to subtract the offset when setting time to RTC * device. Otherwise we need to subtract the offset to make the time * values are valid for RTC hardware device. */ if (secs >= rtc->range_min && secs <= rtc->range_max) return; rtc_time64_to_tm(secs - rtc->offset_secs, tm); } static int rtc_valid_range(struct rtc_device *rtc, struct rtc_time *tm) { if (rtc->range_min != rtc->range_max) { time64_t time = rtc_tm_to_time64(tm); time64_t range_min = rtc->set_start_time ? rtc->start_secs : rtc->range_min; timeu64_t range_max = rtc->set_start_time ? (rtc->start_secs + rtc->range_max - rtc->range_min) : rtc->range_max; if (time < range_min || time > range_max) return -ERANGE; } return 0; } static int __rtc_read_time(struct rtc_device *rtc, struct rtc_time *tm) { int err; if (!rtc->ops) { err = -ENODEV; } else if (!rtc->ops->read_time) { err = -EINVAL; } else { memset(tm, 0, sizeof(struct rtc_time)); err = rtc->ops->read_time(rtc->dev.parent, tm); if (err < 0) { dev_dbg(&rtc->dev, "read_time: fail to read: %d\n", err); return err; } rtc_add_offset(rtc, tm); err = rtc_valid_tm(tm); if (err < 0) dev_dbg(&rtc->dev, "read_time: rtc_time isn't valid\n"); } return err; } int rtc_read_time(struct rtc_device *rtc, struct rtc_time *tm) { int err; err = mutex_lock_interruptible(&rtc->ops_lock); if (err) return err; err = __rtc_read_time(rtc, tm); mutex_unlock(&rtc->ops_lock); trace_rtc_read_time(rtc_tm_to_time64(tm), err); return err; } EXPORT_SYMBOL_GPL(rtc_read_time); int rtc_set_time(struct rtc_device *rtc, struct rtc_time *tm) { int err, uie; err = rtc_valid_tm(tm); if (err != 0) return err; err = rtc_valid_range(rtc, tm); if (err) return err; rtc_subtract_offset(rtc, tm); #ifdef CONFIG_RTC_INTF_DEV_UIE_EMUL uie = rtc->uie_rtctimer.enabled || rtc->uie_irq_active; #else uie = rtc->uie_rtctimer.enabled; #endif if (uie) { err = rtc_update_irq_enable(rtc, 0); if (err) return err; } err = mutex_lock_interruptible(&rtc->ops_lock); if (err) return err; if (!rtc->ops) err = -ENODEV; else if (rtc->ops->set_time) err = rtc->ops->set_time(rtc->dev.parent, tm); else err = -EINVAL; pm_stay_awake(rtc->dev.parent); mutex_unlock(&rtc->ops_lock); /* A timer might have just expired */ schedule_work(&rtc->irqwork); if (uie) { err = rtc_update_irq_enable(rtc, 1); if (err) return err; } trace_rtc_set_time(rtc_tm_to_time64(tm), err); return err; } EXPORT_SYMBOL_GPL(rtc_set_time); static int rtc_read_alarm_internal(struct rtc_device *rtc, struct rtc_wkalrm *alarm) { int err; err = mutex_lock_interruptible(&rtc->ops_lock); if (err) return err; if (!rtc->ops) { err = -ENODEV; } else if (!test_bit(RTC_FEATURE_ALARM, rtc->features) || !rtc->ops->read_alarm) { err = -EINVAL; } else { alarm->enabled = 0; alarm->pending = 0; alarm->time.tm_sec = -1; alarm->time.tm_min = -1; alarm->time.tm_hour = -1; alarm->time.tm_mday = -1; alarm->time.tm_mon = -1; alarm->time.tm_year = -1; alarm->time.tm_wday = -1; alarm->time.tm_yday = -1; alarm->time.tm_isdst = -1; err = rtc->ops->read_alarm(rtc->dev.parent, alarm); } mutex_unlock(&rtc->ops_lock); trace_rtc_read_alarm(err?0:rtc_tm_to_time64(&alarm->time), err); return err; } int __rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm) { int err; struct rtc_time before, now; int first_time = 1; time64_t t_now, t_alm; enum { none, day, month, year } missing = none; unsigned int days; /* The lower level RTC driver may return -1 in some fields, * creating invalid alarm->time values, for reasons like: * * - The hardware may not be capable of filling them in; * many alarms match only on time-of-day fields, not * day/month/year calendar data. * * - Some hardware uses illegal values as "wildcard" match * values, which non-Linux firmware (like a BIOS) may try * to set up as e.g. "alarm 15 minutes after each hour". * Linux uses only oneshot alarms. * * When we see that here, we deal with it by using values from * a current RTC timestamp for any missing (-1) values. The * RTC driver prevents "periodic alarm" modes. * * But this can be racey, because some fields of the RTC timestamp * may have wrapped in the interval since we read the RTC alarm, * which would lead to us inserting inconsistent values in place * of the -1 fields. * * Reading the alarm and timestamp in the reverse sequence * would have the same race condition, and not solve the issue. * * So, we must first read the RTC timestamp, * then read the RTC alarm value, * and then read a second RTC timestamp. * * If any fields of the second timestamp have changed * when compared with the first timestamp, then we know * our timestamp may be inconsistent with that used by * the low-level rtc_read_alarm_internal() function. * * So, when the two timestamps disagree, we just loop and do * the process again to get a fully consistent set of values. * * This could all instead be done in the lower level driver, * but since more than one lower level RTC implementation needs it, * then it's probably best to do it here instead of there.. */ /* Get the "before" timestamp */ err = rtc_read_time(rtc, &before); if (err < 0) return err; do { if (!first_time) memcpy(&before, &now, sizeof(struct rtc_time)); first_time = 0; /* get the RTC alarm values, which may be incomplete */ err = rtc_read_alarm_internal(rtc, alarm); if (err) return err; /* full-function RTCs won't have such missing fields */ err = rtc_valid_tm(&alarm->time); if (!err) goto done; /* get the "after" timestamp, to detect wrapped fields */ err = rtc_read_time(rtc, &now); if (err < 0) return err; /* note that tm_sec is a "don't care" value here: */ } while (before.tm_min != now.tm_min || before.tm_hour != now.tm_hour || before.tm_mon != now.tm_mon || before.tm_year != now.tm_year); /* Fill in the missing alarm fields using the timestamp; we * know there's at least one since alarm->time is invalid. */ if (alarm->time.tm_sec == -1) alarm->time.tm_sec = now.tm_sec; if (alarm->time.tm_min == -1) alarm->time.tm_min = now.tm_min; if (alarm->time.tm_hour == -1) alarm->time.tm_hour = now.tm_hour; /* For simplicity, only support date rollover for now */ if (alarm->time.tm_mday < 1 || alarm->time.tm_mday > 31) { alarm->time.tm_mday = now.tm_mday; missing = day; } if ((unsigned int)alarm->time.tm_mon >= 12) { alarm->time.tm_mon = now.tm_mon; if (missing == none) missing = month; } if (alarm->time.tm_year == -1) { alarm->time.tm_year = now.tm_year; if (missing == none) missing = year; } /* Can't proceed if alarm is still invalid after replacing * missing fields. */ err = rtc_valid_tm(&alarm->time); if (err) goto done; /* with luck, no rollover is needed */ t_now = rtc_tm_to_time64(&now); t_alm = rtc_tm_to_time64(&alarm->time); if (t_now < t_alm) goto done; switch (missing) { /* 24 hour rollover ... if it's now 10am Monday, an alarm that * that will trigger at 5am will do so at 5am Tuesday, which * could also be in the next month or year. This is a common * case, especially for PCs. */ case day: dev_dbg(&rtc->dev, "alarm rollover: %s\n", "day"); t_alm += 24 * 60 * 60; rtc_time64_to_tm(t_alm, &alarm->time); break; /* Month rollover ... if it's the 31th, an alarm on the 3rd will * be next month. An alarm matching on the 30th, 29th, or 28th * may end up in the month after that! Many newer PCs support * this type of alarm. */ case month: dev_dbg(&rtc->dev, "alarm rollover: %s\n", "month"); do { if (alarm->time.tm_mon < 11) { alarm->time.tm_mon++; } else { alarm->time.tm_mon = 0; alarm->time.tm_year++; } days = rtc_month_days(alarm->time.tm_mon, alarm->time.tm_year); } while (days < alarm->time.tm_mday); break; /* Year rollover ... easy except for leap years! */ case year: dev_dbg(&rtc->dev, "alarm rollover: %s\n", "year"); do { alarm->time.tm_year++; } while (!is_leap_year(alarm->time.tm_year + 1900) && rtc_valid_tm(&alarm->time) != 0); break; default: dev_warn(&rtc->dev, "alarm rollover not handled\n"); } err = rtc_valid_tm(&alarm->time); done: if (err && alarm->enabled) dev_warn(&rtc->dev, "invalid alarm value: %ptR\n", &alarm->time); else rtc_add_offset(rtc, &alarm->time); return err; } int rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm) { int err; err = mutex_lock_interruptible(&rtc->ops_lock); if (err) return err; if (!rtc->ops) { err = -ENODEV; } else if (!test_bit(RTC_FEATURE_ALARM, rtc->features)) { err = -EINVAL; } else { memset(alarm, 0, sizeof(struct rtc_wkalrm)); alarm->enabled = rtc->aie_timer.enabled; alarm->time = rtc_ktime_to_tm(rtc->aie_timer.node.expires); } mutex_unlock(&rtc->ops_lock); trace_rtc_read_alarm(rtc_tm_to_time64(&alarm->time), err); return err; } EXPORT_SYMBOL_GPL(rtc_read_alarm); static int __rtc_set_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm) { struct rtc_time tm; time64_t now, scheduled; int err; err = rtc_valid_tm(&alarm->time); if (err) return err; scheduled = rtc_tm_to_time64(&alarm->time); /* Make sure we're not setting alarms in the past */ err = __rtc_read_time(rtc, &tm); if (err) return err; now = rtc_tm_to_time64(&tm); if (scheduled <= now) return -ETIME; /* * XXX - We just checked to make sure the alarm time is not * in the past, but there is still a race window where if * the is alarm set for the next second and the second ticks * over right here, before we set the alarm. */ rtc_subtract_offset(rtc, &alarm->time); if (!rtc->ops) err = -ENODEV; else if (!test_bit(RTC_FEATURE_ALARM, rtc->features)) err = -EINVAL; else err = rtc->ops->set_alarm(rtc->dev.parent, alarm); /* * Check for potential race described above. If the waiting for next * second, and the second just ticked since the check above, either * * 1) It ticked after the alarm was set, and an alarm irq should be * generated. * * 2) It ticked before the alarm was set, and alarm irq most likely will * not be generated. * * While we cannot easily check for which of these two scenarios we * are in, we can return -ETIME to signal that the timer has already * expired, which is true in both cases. */ if ((scheduled - now) <= 1) { err = __rtc_read_time(rtc, &tm); if (err) return err; now = rtc_tm_to_time64(&tm); if (scheduled <= now) return -ETIME; } trace_rtc_set_alarm(rtc_tm_to_time64(&alarm->time), err); return err; } int rtc_set_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm) { ktime_t alarm_time; int err; if (!rtc->ops) return -ENODEV; else if (!test_bit(RTC_FEATURE_ALARM, rtc->features)) return -EINVAL; err = rtc_valid_tm(&alarm->time); if (err != 0) return err; err = rtc_valid_range(rtc, &alarm->time); if (err) return err; err = mutex_lock_interruptible(&rtc->ops_lock); if (err) return err; if (rtc->aie_timer.enabled) rtc_timer_remove(rtc, &rtc->aie_timer); alarm_time = rtc_tm_to_ktime(alarm->time); /* * Round down so we never miss a deadline, checking for past deadline is * done in __rtc_set_alarm */ if (test_bit(RTC_FEATURE_ALARM_RES_MINUTE, rtc->features)) alarm_time = ktime_sub_ns(alarm_time, (u64)alarm->time.tm_sec * NSEC_PER_SEC); rtc->aie_timer.node.expires = alarm_time; rtc->aie_timer.period = 0; if (alarm->enabled) err = rtc_timer_enqueue(rtc, &rtc->aie_timer); mutex_unlock(&rtc->ops_lock); return err; } EXPORT_SYMBOL_GPL(rtc_set_alarm); /* Called once per device from rtc_device_register */ int rtc_initialize_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm) { int err; struct rtc_time now; err = rtc_valid_tm(&alarm->time); if (err != 0) return err; err = rtc_read_time(rtc, &now); if (err) return err; err = mutex_lock_interruptible(&rtc->ops_lock); if (err) return err; rtc->aie_timer.node.expires = rtc_tm_to_ktime(alarm->time); rtc->aie_timer.period = 0; /* Alarm has to be enabled & in the future for us to enqueue it */ if (alarm->enabled && (rtc_tm_to_ktime(now) < rtc->aie_timer.node.expires)) { rtc->aie_timer.enabled = 1; timerqueue_add(&rtc->timerqueue, &rtc->aie_timer.node); trace_rtc_timer_enqueue(&rtc->aie_timer); } mutex_unlock(&rtc->ops_lock); return err; } EXPORT_SYMBOL_GPL(rtc_initialize_alarm); int rtc_alarm_irq_enable(struct rtc_device *rtc, unsigned int enabled) { int err; err = mutex_lock_interruptible(&rtc->ops_lock); if (err) return err; if (rtc->aie_timer.enabled != enabled) { if (enabled) err = rtc_timer_enqueue(rtc, &rtc->aie_timer); else rtc_timer_remove(rtc, &rtc->aie_timer); } if (err) /* nothing */; else if (!rtc->ops) err = -ENODEV; else if (!test_bit(RTC_FEATURE_ALARM, rtc->features) || !rtc->ops->alarm_irq_enable) err = -EINVAL; else err = rtc->ops->alarm_irq_enable(rtc->dev.parent, enabled); mutex_unlock(&rtc->ops_lock); trace_rtc_alarm_irq_enable(enabled, err); return err; } EXPORT_SYMBOL_GPL(rtc_alarm_irq_enable); int rtc_update_irq_enable(struct rtc_device *rtc, unsigned int enabled) { int err; err = mutex_lock_interruptible(&rtc->ops_lock); if (err) return err; #ifdef CONFIG_RTC_INTF_DEV_UIE_EMUL if (enabled == 0 && rtc->uie_irq_active) { mutex_unlock(&rtc->ops_lock); return rtc_dev_update_irq_enable_emul(rtc, 0); } #endif /* make sure we're changing state */ if (rtc->uie_rtctimer.enabled == enabled) goto out; if (!test_bit(RTC_FEATURE_UPDATE_INTERRUPT, rtc->features) || !test_bit(RTC_FEATURE_ALARM, rtc->features)) { mutex_unlock(&rtc->ops_lock); #ifdef CONFIG_RTC_INTF_DEV_UIE_EMUL return rtc_dev_update_irq_enable_emul(rtc, enabled); #else return -EINVAL; #endif } if (enabled) { struct rtc_time tm; ktime_t now, onesec; err = __rtc_read_time(rtc, &tm); if (err) goto out; onesec = ktime_set(1, 0); now = rtc_tm_to_ktime(tm); rtc->uie_rtctimer.node.expires = ktime_add(now, onesec); rtc->uie_rtctimer.period = ktime_set(1, 0); err = rtc_timer_enqueue(rtc, &rtc->uie_rtctimer); if (!err && rtc->ops && rtc->ops->alarm_irq_enable) err = rtc->ops->alarm_irq_enable(rtc->dev.parent, 1); if (err) goto out; } else { rtc_timer_remove(rtc, &rtc->uie_rtctimer); } out: mutex_unlock(&rtc->ops_lock); return err; } EXPORT_SYMBOL_GPL(rtc_update_irq_enable); /** * rtc_handle_legacy_irq - AIE, UIE and PIE event hook * @rtc: pointer to the rtc device * @num: number of occurence of the event * @mode: type of the event, RTC_AF, RTC_UF of RTC_PF * * This function is called when an AIE, UIE or PIE mode interrupt * has occurred (or been emulated). * */ void rtc_handle_legacy_irq(struct rtc_device *rtc, int num, int mode) { unsigned long flags; /* mark one irq of the appropriate mode */ spin_lock_irqsave(&rtc->irq_lock, flags); rtc->irq_data = (rtc->irq_data + (num << 8)) | (RTC_IRQF | mode); spin_unlock_irqrestore(&rtc->irq_lock, flags); wake_up_interruptible(&rtc->irq_queue); kill_fasync(&rtc->async_queue, SIGIO, POLL_IN); } /** * rtc_aie_update_irq - AIE mode rtctimer hook * @rtc: pointer to the rtc_device * * This functions is called when the aie_timer expires. */ void rtc_aie_update_irq(struct rtc_device *rtc) { rtc_handle_legacy_irq(rtc, 1, RTC_AF); } /** * rtc_uie_update_irq - UIE mode rtctimer hook * @rtc: pointer to the rtc_device * * This functions is called when the uie_timer expires. */ void rtc_uie_update_irq(struct rtc_device *rtc) { rtc_handle_legacy_irq(rtc, 1, RTC_UF); } /** * rtc_pie_update_irq - PIE mode hrtimer hook * @timer: pointer to the pie mode hrtimer * * This function is used to emulate PIE mode interrupts * using an hrtimer. This function is called when the periodic * hrtimer expires. */ enum hrtimer_restart rtc_pie_update_irq(struct hrtimer *timer) { struct rtc_device *rtc; ktime_t period; u64 count; rtc = container_of(timer, struct rtc_device, pie_timer); period = NSEC_PER_SEC / rtc->irq_freq; count = hrtimer_forward_now(timer, period); rtc_handle_legacy_irq(rtc, count, RTC_PF); return HRTIMER_RESTART; } /** * rtc_update_irq - Triggered when a RTC interrupt occurs. * @rtc: the rtc device * @num: how many irqs are being reported (usually one) * @events: mask of RTC_IRQF with one or more of RTC_PF, RTC_AF, RTC_UF * Context: any */ void rtc_update_irq(struct rtc_device *rtc, unsigned long num, unsigned long events) { if (IS_ERR_OR_NULL(rtc)) return; pm_stay_awake(rtc->dev.parent); schedule_work(&rtc->irqwork); } EXPORT_SYMBOL_GPL(rtc_update_irq); struct rtc_device *rtc_class_open(const char *name) { struct device *dev; struct rtc_device *rtc = NULL; dev = class_find_device_by_name(&rtc_class, name); if (dev) rtc = to_rtc_device(dev); if (rtc) { if (!try_module_get(rtc->owner)) { put_device(dev); rtc = NULL; } } return rtc; } EXPORT_SYMBOL_GPL(rtc_class_open); void rtc_class_close(struct rtc_device *rtc) { module_put(rtc->owner); put_device(&rtc->dev); } EXPORT_SYMBOL_GPL(rtc_class_close); static int rtc_update_hrtimer(struct rtc_device *rtc, int enabled) { /* * We always cancel the timer here first, because otherwise * we could run into BUG_ON(timer->state != HRTIMER_STATE_CALLBACK); * when we manage to start the timer before the callback * returns HRTIMER_RESTART. * * We cannot use hrtimer_cancel() here as a running callback * could be blocked on rtc->irq_task_lock and hrtimer_cancel() * would spin forever. */ if (hrtimer_try_to_cancel(&rtc->pie_timer) < 0) return -1; if (enabled) { ktime_t period = NSEC_PER_SEC / rtc->irq_freq; hrtimer_start(&rtc->pie_timer, period, HRTIMER_MODE_REL); } return 0; } /** * rtc_irq_set_state - enable/disable 2^N Hz periodic IRQs * @rtc: the rtc device * @enabled: true to enable periodic IRQs * Context: any * * Note that rtc_irq_set_freq() should previously have been used to * specify the desired frequency of periodic IRQ. */ int rtc_irq_set_state(struct rtc_device *rtc, int enabled) { int err = 0; while (rtc_update_hrtimer(rtc, enabled) < 0) cpu_relax(); rtc->pie_enabled = enabled; trace_rtc_irq_set_state(enabled, err); return err; } /** * rtc_irq_set_freq - set 2^N Hz periodic IRQ frequency for IRQ * @rtc: the rtc device * @freq: positive frequency * Context: any * * Note that rtc_irq_set_state() is used to enable or disable the * periodic IRQs. */ int rtc_irq_set_freq(struct rtc_device *rtc, int freq) { int err = 0; if (freq <= 0 || freq > RTC_MAX_FREQ) return -EINVAL; rtc->irq_freq = freq; while (rtc->pie_enabled && rtc_update_hrtimer(rtc, 1) < 0) cpu_relax(); trace_rtc_irq_set_freq(freq, err); return err; } /** * rtc_timer_enqueue - Adds a rtc_timer to the rtc_device timerqueue * @rtc: rtc device * @timer: timer being added. * * Enqueues a timer onto the rtc devices timerqueue and sets * the next alarm event appropriately. * * Sets the enabled bit on the added timer. * * Must hold ops_lock for proper serialization of timerqueue */ static int rtc_timer_enqueue(struct rtc_device *rtc, struct rtc_timer *timer) { struct timerqueue_node *next = timerqueue_getnext(&rtc->timerqueue); struct rtc_time tm; ktime_t now; int err; err = __rtc_read_time(rtc, &tm); if (err) return err; timer->enabled = 1; now = rtc_tm_to_ktime(tm); /* Skip over expired timers */ while (next) { if (next->expires >= now) break; next = timerqueue_iterate_next(next); } timerqueue_add(&rtc->timerqueue, &timer->node); trace_rtc_timer_enqueue(timer); if (!next || ktime_before(timer->node.expires, next->expires)) { struct rtc_wkalrm alarm; alarm.time = rtc_ktime_to_tm(timer->node.expires); alarm.enabled = 1; err = __rtc_set_alarm(rtc, &alarm); if (err == -ETIME) { pm_stay_awake(rtc->dev.parent); schedule_work(&rtc->irqwork); } else if (err) { timerqueue_del(&rtc->timerqueue, &timer->node); trace_rtc_timer_dequeue(timer); timer->enabled = 0; return err; } } return 0; } static void rtc_alarm_disable(struct rtc_device *rtc) { if (!rtc->ops || !test_bit(RTC_FEATURE_ALARM, rtc->features) || !rtc->ops->alarm_irq_enable) return; rtc->ops->alarm_irq_enable(rtc->dev.parent, false); trace_rtc_alarm_irq_enable(0, 0); } /** * rtc_timer_remove - Removes a rtc_timer from the rtc_device timerqueue * @rtc: rtc device * @timer: timer being removed. * * Removes a timer onto the rtc devices timerqueue and sets * the next alarm event appropriately. * * Clears the enabled bit on the removed timer. * * Must hold ops_lock for proper serialization of timerqueue */ static void rtc_timer_remove(struct rtc_device *rtc, struct rtc_timer *timer) { struct timerqueue_node *next = timerqueue_getnext(&rtc->timerqueue); timerqueue_del(&rtc->timerqueue, &timer->node); trace_rtc_timer_dequeue(timer); timer->enabled = 0; if (next == &timer->node) { struct rtc_wkalrm alarm; int err; next = timerqueue_getnext(&rtc->timerqueue); if (!next) { rtc_alarm_disable(rtc); return; } alarm.time = rtc_ktime_to_tm(next->expires); alarm.enabled = 1; err = __rtc_set_alarm(rtc, &alarm); if (err == -ETIME) { pm_stay_awake(rtc->dev.parent); schedule_work(&rtc->irqwork); } } } /** * rtc_timer_do_work - Expires rtc timers * @work: work item * * Expires rtc timers. Reprograms next alarm event if needed. * Called via worktask. * * Serializes access to timerqueue via ops_lock mutex */ void rtc_timer_do_work(struct work_struct *work) { struct rtc_timer *timer; struct timerqueue_node *next; ktime_t now; struct rtc_time tm; int err; struct rtc_device *rtc = container_of(work, struct rtc_device, irqwork); mutex_lock(&rtc->ops_lock); again: err = __rtc_read_time(rtc, &tm); if (err) { mutex_unlock(&rtc->ops_lock); return; } now = rtc_tm_to_ktime(tm); while ((next = timerqueue_getnext(&rtc->timerqueue))) { if (next->expires > now) break; /* expire timer */ timer = container_of(next, struct rtc_timer, node); timerqueue_del(&rtc->timerqueue, &timer->node); trace_rtc_timer_dequeue(timer); timer->enabled = 0; if (timer->func) timer->func(timer->rtc); trace_rtc_timer_fired(timer); /* Re-add/fwd periodic timers */ if (ktime_to_ns(timer->period)) { timer->node.expires = ktime_add(timer->node.expires, timer->period); timer->enabled = 1; timerqueue_add(&rtc->timerqueue, &timer->node); trace_rtc_timer_enqueue(timer); } } /* Set next alarm */ if (next) { struct rtc_wkalrm alarm; int err; int retry = 3; alarm.time = rtc_ktime_to_tm(next->expires); alarm.enabled = 1; reprogram: err = __rtc_set_alarm(rtc, &alarm); if (err == -ETIME) { goto again; } else if (err) { if (retry-- > 0) goto reprogram; timer = container_of(next, struct rtc_timer, node); timerqueue_del(&rtc->timerqueue, &timer->node); trace_rtc_timer_dequeue(timer); timer->enabled = 0; dev_err(&rtc->dev, "__rtc_set_alarm: err=%d\n", err); goto again; } } else { rtc_alarm_disable(rtc); } pm_relax(rtc->dev.parent); mutex_unlock(&rtc->ops_lock); } /* rtc_timer_init - Initializes an rtc_timer * @timer: timer to be intiialized * @f: function pointer to be called when timer fires * @rtc: pointer to the rtc_device * * Kernel interface to initializing an rtc_timer. */ void rtc_timer_init(struct rtc_timer *timer, void (*f)(struct rtc_device *r), struct rtc_device *rtc) { timerqueue_init(&timer->node); timer->enabled = 0; timer->func = f; timer->rtc = rtc; } /* rtc_timer_start - Sets an rtc_timer to fire in the future * @ rtc: rtc device to be used * @ timer: timer being set * @ expires: time at which to expire the timer * @ period: period that the timer will recur * * Kernel interface to set an rtc_timer */ int rtc_timer_start(struct rtc_device *rtc, struct rtc_timer *timer, ktime_t expires, ktime_t period) { int ret = 0; mutex_lock(&rtc->ops_lock); if (timer->enabled) rtc_timer_remove(rtc, timer); timer->node.expires = expires; timer->period = period; ret = rtc_timer_enqueue(rtc, timer); mutex_unlock(&rtc->ops_lock); return ret; } /* rtc_timer_cancel - Stops an rtc_timer * @ rtc: rtc device to be used * @ timer: timer being set * * Kernel interface to cancel an rtc_timer */ void rtc_timer_cancel(struct rtc_device *rtc, struct rtc_timer *timer) { mutex_lock(&rtc->ops_lock); if (timer->enabled) rtc_timer_remove(rtc, timer); mutex_unlock(&rtc->ops_lock); } /** * rtc_read_offset - Read the amount of rtc offset in parts per billion * @rtc: rtc device to be used * @offset: the offset in parts per billion * * see below for details. * * Kernel interface to read rtc clock offset * Returns 0 on success, or a negative number on error. * If read_offset() is not implemented for the rtc, return -EINVAL */ int rtc_read_offset(struct rtc_device *rtc, long *offset) { int ret; if (!rtc->ops) return -ENODEV; if (!rtc->ops->read_offset) return -EINVAL; mutex_lock(&rtc->ops_lock); ret = rtc->ops->read_offset(rtc->dev.parent, offset); mutex_unlock(&rtc->ops_lock); trace_rtc_read_offset(*offset, ret); return ret; } /** * rtc_set_offset - Adjusts the duration of the average second * @rtc: rtc device to be used * @offset: the offset in parts per billion * * Some rtc's allow an adjustment to the average duration of a second * to compensate for differences in the actual clock rate due to temperature, * the crystal, capacitor, etc. * * The adjustment applied is as follows: * t = t0 * (1 + offset * 1e-9) * where t0 is the measured length of 1 RTC second with offset = 0 * * Kernel interface to adjust an rtc clock offset. * Return 0 on success, or a negative number on error. * If the rtc offset is not setable (or not implemented), return -EINVAL */ int rtc_set_offset(struct rtc_device *rtc, long offset) { int ret; if (!rtc->ops) return -ENODEV; if (!rtc->ops->set_offset) return -EINVAL; mutex_lock(&rtc->ops_lock); ret = rtc->ops->set_offset(rtc->dev.parent, offset); mutex_unlock(&rtc->ops_lock); trace_rtc_set_offset(offset, ret); return ret; } |
| 453 4 14 20 130 488 2 4 132 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 | /* SPDX-License-Identifier: GPL-2.0 */ #if !defined(_TRACE_KVM_MAIN_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_KVM_MAIN_H #include <linux/tracepoint.h> #undef TRACE_SYSTEM #define TRACE_SYSTEM kvm #define ERSN(x) { KVM_EXIT_##x, "KVM_EXIT_" #x } #define kvm_trace_exit_reason \ ERSN(UNKNOWN), ERSN(EXCEPTION), ERSN(IO), ERSN(HYPERCALL), \ ERSN(DEBUG), ERSN(HLT), ERSN(MMIO), ERSN(IRQ_WINDOW_OPEN), \ ERSN(SHUTDOWN), ERSN(FAIL_ENTRY), ERSN(INTR), ERSN(SET_TPR), \ ERSN(TPR_ACCESS), ERSN(S390_SIEIC), ERSN(S390_RESET), ERSN(DCR),\ ERSN(NMI), ERSN(INTERNAL_ERROR), ERSN(OSI), ERSN(PAPR_HCALL), \ ERSN(S390_UCONTROL), ERSN(WATCHDOG), ERSN(S390_TSCH), ERSN(EPR),\ ERSN(SYSTEM_EVENT), ERSN(S390_STSI), ERSN(IOAPIC_EOI), \ ERSN(HYPERV), ERSN(ARM_NISV), ERSN(X86_RDMSR), ERSN(X86_WRMSR) TRACE_EVENT(kvm_userspace_exit, TP_PROTO(__u32 reason, int errno), TP_ARGS(reason, errno), TP_STRUCT__entry( __field( __u32, reason ) __field( int, errno ) ), TP_fast_assign( __entry->reason = reason; __entry->errno = errno; ), TP_printk("reason %s (%d)", __entry->errno < 0 ? (__entry->errno == -EINTR ? "restart" : "error") : __print_symbolic(__entry->reason, kvm_trace_exit_reason), __entry->errno < 0 ? -__entry->errno : __entry->reason) ); TRACE_EVENT(kvm_vcpu_wakeup, TP_PROTO(__u64 ns, bool waited, bool valid), TP_ARGS(ns, waited, valid), TP_STRUCT__entry( __field( __u64, ns ) __field( bool, waited ) __field( bool, valid ) ), TP_fast_assign( __entry->ns = ns; __entry->waited = waited; __entry->valid = valid; ), TP_printk("%s time %lld ns, polling %s", __entry->waited ? "wait" : "poll", __entry->ns, __entry->valid ? "valid" : "invalid") ); #if defined(CONFIG_HAVE_KVM_IRQCHIP) TRACE_EVENT(kvm_set_irq, TP_PROTO(unsigned int gsi, int level, int irq_source_id), TP_ARGS(gsi, level, irq_source_id), TP_STRUCT__entry( __field( unsigned int, gsi ) __field( int, level ) __field( int, irq_source_id ) ), TP_fast_assign( __entry->gsi = gsi; __entry->level = level; __entry->irq_source_id = irq_source_id; ), TP_printk("gsi %u level %d source %d", __entry->gsi, __entry->level, __entry->irq_source_id) ); #ifdef CONFIG_KVM_IOAPIC #define kvm_irqchips \ {KVM_IRQCHIP_PIC_MASTER, "PIC master"}, \ {KVM_IRQCHIP_PIC_SLAVE, "PIC slave"}, \ {KVM_IRQCHIP_IOAPIC, "IOAPIC"} #endif /* CONFIG_KVM_IOAPIC */ #ifdef kvm_irqchips #define kvm_ack_irq_string "irqchip %s pin %u" #define kvm_ack_irq_parm __print_symbolic(__entry->irqchip, kvm_irqchips), __entry->pin #else #define kvm_ack_irq_string "irqchip %d pin %u" #define kvm_ack_irq_parm __entry->irqchip, __entry->pin #endif TRACE_EVENT(kvm_ack_irq, TP_PROTO(unsigned int irqchip, unsigned int pin), TP_ARGS(irqchip, pin), TP_STRUCT__entry( __field( unsigned int, irqchip ) __field( unsigned int, pin ) ), TP_fast_assign( __entry->irqchip = irqchip; __entry->pin = pin; ), TP_printk(kvm_ack_irq_string, kvm_ack_irq_parm) ); #endif /* defined(CONFIG_HAVE_KVM_IRQCHIP) */ #define KVM_TRACE_MMIO_READ_UNSATISFIED 0 #define KVM_TRACE_MMIO_READ 1 #define KVM_TRACE_MMIO_WRITE 2 #define kvm_trace_symbol_mmio \ { KVM_TRACE_MMIO_READ_UNSATISFIED, "unsatisfied-read" }, \ { KVM_TRACE_MMIO_READ, "read" }, \ { KVM_TRACE_MMIO_WRITE, "write" } TRACE_EVENT(kvm_mmio, TP_PROTO(int type, int len, u64 gpa, void *val), TP_ARGS(type, len, gpa, val), TP_STRUCT__entry( __field( u32, type ) __field( u32, len ) __field( u64, gpa ) __field( u64, val ) ), TP_fast_assign( __entry->type = type; __entry->len = len; __entry->gpa = gpa; __entry->val = 0; if (val) memcpy(&__entry->val, val, min_t(u32, sizeof(__entry->val), len)); ), TP_printk("mmio %s len %u gpa 0x%llx val 0x%llx", __print_symbolic(__entry->type, kvm_trace_symbol_mmio), __entry->len, __entry->gpa, __entry->val) ); #define kvm_fpu_load_symbol \ {0, "unload"}, \ {1, "load"} TRACE_EVENT(kvm_fpu, TP_PROTO(int load), TP_ARGS(load), TP_STRUCT__entry( __field( u32, load ) ), TP_fast_assign( __entry->load = load; ), TP_printk("%s", __print_symbolic(__entry->load, kvm_fpu_load_symbol)) ); #ifdef CONFIG_KVM_ASYNC_PF DECLARE_EVENT_CLASS(kvm_async_get_page_class, TP_PROTO(u64 gva, u64 gfn), TP_ARGS(gva, gfn), TP_STRUCT__entry( __field(__u64, gva) __field(u64, gfn) ), TP_fast_assign( __entry->gva = gva; __entry->gfn = gfn; ), TP_printk("gva = %#llx, gfn = %#llx", __entry->gva, __entry->gfn) ); DEFINE_EVENT(kvm_async_get_page_class, kvm_try_async_get_page, TP_PROTO(u64 gva, u64 gfn), TP_ARGS(gva, gfn) ); DEFINE_EVENT(kvm_async_get_page_class, kvm_async_pf_repeated_fault, TP_PROTO(u64 gva, u64 gfn), TP_ARGS(gva, gfn) ); DECLARE_EVENT_CLASS(kvm_async_pf_nopresent_ready, TP_PROTO(u64 token, u64 gva), TP_ARGS(token, gva), TP_STRUCT__entry( __field(__u64, token) __field(__u64, gva) ), TP_fast_assign( __entry->token = token; __entry->gva = gva; ), TP_printk("token %#llx gva %#llx", __entry->token, __entry->gva) ); DEFINE_EVENT(kvm_async_pf_nopresent_ready, kvm_async_pf_not_present, TP_PROTO(u64 token, u64 gva), TP_ARGS(token, gva) ); DEFINE_EVENT(kvm_async_pf_nopresent_ready, kvm_async_pf_ready, TP_PROTO(u64 token, u64 gva), TP_ARGS(token, gva) ); TRACE_EVENT( kvm_async_pf_completed, TP_PROTO(unsigned long address, u64 gva), TP_ARGS(address, gva), TP_STRUCT__entry( __field(unsigned long, address) __field(u64, gva) ), TP_fast_assign( __entry->address = address; __entry->gva = gva; ), TP_printk("gva %#llx address %#lx", __entry->gva, __entry->address) ); #endif TRACE_EVENT(kvm_halt_poll_ns, TP_PROTO(bool grow, unsigned int vcpu_id, unsigned int new, unsigned int old), TP_ARGS(grow, vcpu_id, new, old), TP_STRUCT__entry( __field(bool, grow) __field(unsigned int, vcpu_id) __field(unsigned int, new) __field(unsigned int, old) ), TP_fast_assign( __entry->grow = grow; __entry->vcpu_id = vcpu_id; __entry->new = new; __entry->old = old; ), TP_printk("vcpu %u: halt_poll_ns %u (%s %u)", __entry->vcpu_id, __entry->new, __entry->grow ? "grow" : "shrink", __entry->old) ); #define trace_kvm_halt_poll_ns_grow(vcpu_id, new, old) \ trace_kvm_halt_poll_ns(true, vcpu_id, new, old) #define trace_kvm_halt_poll_ns_shrink(vcpu_id, new, old) \ trace_kvm_halt_poll_ns(false, vcpu_id, new, old) TRACE_EVENT(kvm_dirty_ring_push, TP_PROTO(struct kvm_dirty_ring *ring, u32 slot, u64 offset), TP_ARGS(ring, slot, offset), TP_STRUCT__entry( __field(int, index) __field(u32, dirty_index) __field(u32, reset_index) __field(u32, slot) __field(u64, offset) ), TP_fast_assign( __entry->index = ring->index; __entry->dirty_index = ring->dirty_index; __entry->reset_index = ring->reset_index; __entry->slot = slot; __entry->offset = offset; ), TP_printk("ring %d: dirty 0x%x reset 0x%x " "slot %u offset 0x%llx (used %u)", __entry->index, __entry->dirty_index, __entry->reset_index, __entry->slot, __entry->offset, __entry->dirty_index - __entry->reset_index) ); TRACE_EVENT(kvm_dirty_ring_reset, TP_PROTO(struct kvm_dirty_ring *ring), TP_ARGS(ring), TP_STRUCT__entry( __field(int, index) __field(u32, dirty_index) __field(u32, reset_index) ), TP_fast_assign( __entry->index = ring->index; __entry->dirty_index = ring->dirty_index; __entry->reset_index = ring->reset_index; ), TP_printk("ring %d: dirty 0x%x reset 0x%x (used %u)", __entry->index, __entry->dirty_index, __entry->reset_index, __entry->dirty_index - __entry->reset_index) ); TRACE_EVENT(kvm_dirty_ring_exit, TP_PROTO(struct kvm_vcpu *vcpu), TP_ARGS(vcpu), TP_STRUCT__entry( __field(int, vcpu_id) ), TP_fast_assign( __entry->vcpu_id = vcpu->vcpu_id; ), TP_printk("vcpu %d", __entry->vcpu_id) ); #ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES /* * @start: Starting address of guest memory range * @end: End address of guest memory range * @attr: The value of the attribute being set. */ TRACE_EVENT(kvm_vm_set_mem_attributes, TP_PROTO(gfn_t start, gfn_t end, unsigned long attr), TP_ARGS(start, end, attr), TP_STRUCT__entry( __field(gfn_t, start) __field(gfn_t, end) __field(unsigned long, attr) ), TP_fast_assign( __entry->start = start; __entry->end = end; __entry->attr = attr; ), TP_printk("%#016llx -- %#016llx [0x%lx]", __entry->start, __entry->end, __entry->attr) ); #endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ TRACE_EVENT(kvm_unmap_hva_range, TP_PROTO(unsigned long start, unsigned long end), TP_ARGS(start, end), TP_STRUCT__entry( __field( unsigned long, start ) __field( unsigned long, end ) ), TP_fast_assign( __entry->start = start; __entry->end = end; ), TP_printk("mmu notifier unmap range: %#016lx -- %#016lx", __entry->start, __entry->end) ); TRACE_EVENT(kvm_age_hva, TP_PROTO(unsigned long start, unsigned long end), TP_ARGS(start, end), TP_STRUCT__entry( __field( unsigned long, start ) __field( unsigned long, end ) ), TP_fast_assign( __entry->start = start; __entry->end = end; ), TP_printk("mmu notifier age hva: %#016lx -- %#016lx", __entry->start, __entry->end) ); TRACE_EVENT(kvm_test_age_hva, TP_PROTO(unsigned long hva), TP_ARGS(hva), TP_STRUCT__entry( __field( unsigned long, hva ) ), TP_fast_assign( __entry->hva = hva; ), TP_printk("mmu notifier test age hva: %#016lx", __entry->hva) ); #endif /* _TRACE_KVM_MAIN_H */ /* This part must be outside protection */ #include <trace/define_trace.h> |
| 5 789 55 54 55 57 110 57 77 25 33 1 1 2 2699 2846 69 884 389 453 95 412 94 415 86 55 78 19 13 2956 2882 489 61 2 32 218 180 106 4 3269 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 | /* SPDX-License-Identifier: GPL-2.0-only */ /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com */ #ifndef _LINUX_BPF_H #define _LINUX_BPF_H 1 #include <uapi/linux/bpf.h> #include <uapi/linux/filter.h> #include <crypto/sha2.h> #include <linux/workqueue.h> #include <linux/file.h> #include <linux/percpu.h> #include <linux/err.h> #include <linux/rbtree_latch.h> #include <linux/numa.h> #include <linux/mm_types.h> #include <linux/wait.h> #include <linux/refcount.h> #include <linux/mutex.h> #include <linux/module.h> #include <linux/kallsyms.h> #include <linux/capability.h> #include <linux/sched/mm.h> #include <linux/slab.h> #include <linux/percpu-refcount.h> #include <linux/stddef.h> #include <linux/bpfptr.h> #include <linux/btf.h> #include <linux/rcupdate_trace.h> #include <linux/static_call.h> #include <linux/memcontrol.h> #include <linux/cfi.h> #include <asm/rqspinlock.h> struct bpf_verifier_env; struct bpf_verifier_log; struct perf_event; struct bpf_prog; struct bpf_prog_aux; struct bpf_map; struct bpf_arena; struct sock; struct seq_file; struct btf; struct btf_type; struct exception_table_entry; struct seq_operations; struct bpf_iter_aux_info; struct bpf_local_storage; struct bpf_local_storage_map; struct kobject; struct mem_cgroup; struct module; struct bpf_func_state; struct ftrace_ops; struct cgroup; struct bpf_token; struct user_namespace; struct super_block; struct inode; extern struct idr btf_idr; extern spinlock_t btf_idr_lock; extern struct kobject *btf_kobj; extern struct bpf_mem_alloc bpf_global_ma, bpf_global_percpu_ma; extern bool bpf_global_ma_set; typedef u64 (*bpf_callback_t)(u64, u64, u64, u64, u64); typedef int (*bpf_iter_init_seq_priv_t)(void *private_data, struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_fini_seq_priv_t)(void *private_data); typedef unsigned int (*bpf_func_t)(const void *, const struct bpf_insn *); struct bpf_iter_seq_info { const struct seq_operations *seq_ops; bpf_iter_init_seq_priv_t init_seq_private; bpf_iter_fini_seq_priv_t fini_seq_private; u32 seq_priv_size; }; /* map is generic key/value storage optionally accessible by eBPF programs */ struct bpf_map_ops { /* funcs callable from userspace (via syscall) */ int (*map_alloc_check)(union bpf_attr *attr); struct bpf_map *(*map_alloc)(union bpf_attr *attr); void (*map_release)(struct bpf_map *map, struct file *map_file); void (*map_free)(struct bpf_map *map); int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); void (*map_release_uref)(struct bpf_map *map); void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); int (*map_lookup_batch)(struct bpf_map *map, const union bpf_attr *attr, union bpf_attr __user *uattr); int (*map_lookup_and_delete_elem)(struct bpf_map *map, void *key, void *value, u64 flags); int (*map_lookup_and_delete_batch)(struct bpf_map *map, const union bpf_attr *attr, union bpf_attr __user *uattr); int (*map_update_batch)(struct bpf_map *map, struct file *map_file, const union bpf_attr *attr, union bpf_attr __user *uattr); int (*map_delete_batch)(struct bpf_map *map, const union bpf_attr *attr, union bpf_attr __user *uattr); /* funcs callable from userspace and from eBPF programs */ void *(*map_lookup_elem)(struct bpf_map *map, void *key); long (*map_update_elem)(struct bpf_map *map, void *key, void *value, u64 flags); long (*map_delete_elem)(struct bpf_map *map, void *key); long (*map_push_elem)(struct bpf_map *map, void *value, u64 flags); long (*map_pop_elem)(struct bpf_map *map, void *value); long (*map_peek_elem)(struct bpf_map *map, void *value); void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu); int (*map_get_hash)(struct bpf_map *map, u32 hash_buf_size, void *hash_buf); /* funcs called by prog_array and perf_event_array map */ void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, int fd); /* If need_defer is true, the implementation should guarantee that * the to-be-put element is still alive before the bpf program, which * may manipulate it, exists. */ void (*map_fd_put_ptr)(struct bpf_map *map, void *ptr, bool need_defer); int (*map_gen_lookup)(struct bpf_map *map, struct bpf_insn *insn_buf); u32 (*map_fd_sys_lookup_elem)(void *ptr); void (*map_seq_show_elem)(struct bpf_map *map, void *key, struct seq_file *m); int (*map_check_btf)(const struct bpf_map *map, const struct btf *btf, const struct btf_type *key_type, const struct btf_type *value_type); /* Prog poke tracking helpers. */ int (*map_poke_track)(struct bpf_map *map, struct bpf_prog_aux *aux); void (*map_poke_untrack)(struct bpf_map *map, struct bpf_prog_aux *aux); void (*map_poke_run)(struct bpf_map *map, u32 key, struct bpf_prog *old, struct bpf_prog *new); /* Direct value access helpers. */ int (*map_direct_value_addr)(const struct bpf_map *map, u64 *imm, u32 off); int (*map_direct_value_meta)(const struct bpf_map *map, u64 imm, u32 *off); int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, struct poll_table_struct *pts); unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); /* Functions called by bpf_local_storage maps */ int (*map_local_storage_charge)(struct bpf_local_storage_map *smap, void *owner, u32 size); void (*map_local_storage_uncharge)(struct bpf_local_storage_map *smap, void *owner, u32 size); struct bpf_local_storage __rcu ** (*map_owner_storage_ptr)(void *owner); /* Misc helpers.*/ long (*map_redirect)(struct bpf_map *map, u64 key, u64 flags); /* map_meta_equal must be implemented for maps that can be * used as an inner map. It is a runtime check to ensure * an inner map can be inserted to an outer map. * * Some properties of the inner map has been used during the * verification time. When inserting an inner map at the runtime, * map_meta_equal has to ensure the inserting map has the same * properties that the verifier has used earlier. */ bool (*map_meta_equal)(const struct bpf_map *meta0, const struct bpf_map *meta1); int (*map_set_for_each_callback_args)(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee); long (*map_for_each_callback)(struct bpf_map *map, bpf_callback_t callback_fn, void *callback_ctx, u64 flags); u64 (*map_mem_usage)(const struct bpf_map *map); /* BTF id of struct allocated by map_alloc */ int *map_btf_id; /* bpf_iter info used to open a seq_file */ const struct bpf_iter_seq_info *iter_seq_info; }; enum { /* Support at most 11 fields in a BTF type */ BTF_FIELDS_MAX = 11, }; enum btf_field_type { BPF_SPIN_LOCK = (1 << 0), BPF_TIMER = (1 << 1), BPF_KPTR_UNREF = (1 << 2), BPF_KPTR_REF = (1 << 3), BPF_KPTR_PERCPU = (1 << 4), BPF_KPTR = BPF_KPTR_UNREF | BPF_KPTR_REF | BPF_KPTR_PERCPU, BPF_LIST_HEAD = (1 << 5), BPF_LIST_NODE = (1 << 6), BPF_RB_ROOT = (1 << 7), BPF_RB_NODE = (1 << 8), BPF_GRAPH_NODE = BPF_RB_NODE | BPF_LIST_NODE, BPF_GRAPH_ROOT = BPF_RB_ROOT | BPF_LIST_HEAD, BPF_REFCOUNT = (1 << 9), BPF_WORKQUEUE = (1 << 10), BPF_UPTR = (1 << 11), BPF_RES_SPIN_LOCK = (1 << 12), BPF_TASK_WORK = (1 << 13), }; enum bpf_cgroup_storage_type { BPF_CGROUP_STORAGE_SHARED, BPF_CGROUP_STORAGE_PERCPU, __BPF_CGROUP_STORAGE_MAX #define MAX_BPF_CGROUP_STORAGE_TYPE __BPF_CGROUP_STORAGE_MAX }; #ifdef CONFIG_CGROUP_BPF # define for_each_cgroup_storage_type(stype) \ for (stype = 0; stype < MAX_BPF_CGROUP_STORAGE_TYPE; stype++) #else # define for_each_cgroup_storage_type(stype) for (; false; ) #endif /* CONFIG_CGROUP_BPF */ typedef void (*btf_dtor_kfunc_t)(void *); struct btf_field_kptr { struct btf *btf; struct module *module; /* dtor used if btf_is_kernel(btf), otherwise the type is * program-allocated, dtor is NULL, and __bpf_obj_drop_impl is used */ btf_dtor_kfunc_t dtor; u32 btf_id; }; struct btf_field_graph_root { struct btf *btf; u32 value_btf_id; u32 node_offset; struct btf_record *value_rec; }; struct btf_field { u32 offset; u32 size; enum btf_field_type type; union { struct btf_field_kptr kptr; struct btf_field_graph_root graph_root; }; }; struct btf_record { u32 cnt; u32 field_mask; int spin_lock_off; int res_spin_lock_off; int timer_off; int wq_off; int refcount_off; int task_work_off; struct btf_field fields[]; }; /* Non-opaque version of bpf_rb_node in uapi/linux/bpf.h */ struct bpf_rb_node_kern { struct rb_node rb_node; void *owner; } __attribute__((aligned(8))); /* Non-opaque version of bpf_list_node in uapi/linux/bpf.h */ struct bpf_list_node_kern { struct list_head list_head; void *owner; } __attribute__((aligned(8))); /* 'Ownership' of program-containing map is claimed by the first program * that is going to use this map or by the first program which FD is * stored in the map to make sure that all callers and callees have the * same prog type, JITed flag and xdp_has_frags flag. */ struct bpf_map_owner { enum bpf_prog_type type; bool jited; bool xdp_has_frags; u64 storage_cookie[MAX_BPF_CGROUP_STORAGE_TYPE]; const struct btf_type *attach_func_proto; enum bpf_attach_type expected_attach_type; }; struct bpf_map { u8 sha[SHA256_DIGEST_SIZE]; const struct bpf_map_ops *ops; struct bpf_map *inner_map_meta; #ifdef CONFIG_SECURITY void *security; #endif enum bpf_map_type map_type; u32 key_size; u32 value_size; u32 max_entries; u64 map_extra; /* any per-map-type extra fields */ u32 map_flags; u32 id; struct btf_record *record; int numa_node; u32 btf_key_type_id; u32 btf_value_type_id; u32 btf_vmlinux_value_type_id; struct btf *btf; #ifdef CONFIG_MEMCG struct obj_cgroup *objcg; #endif char name[BPF_OBJ_NAME_LEN]; struct mutex freeze_mutex; atomic64_t refcnt; atomic64_t usercnt; /* rcu is used before freeing and work is only used during freeing */ union { struct work_struct work; struct rcu_head rcu; }; atomic64_t writecnt; spinlock_t owner_lock; struct bpf_map_owner *owner; bool bypass_spec_v1; bool frozen; /* write-once; write-protected by freeze_mutex */ bool free_after_mult_rcu_gp; bool free_after_rcu_gp; atomic64_t sleepable_refcnt; s64 __percpu *elem_count; u64 cookie; /* write-once */ char *excl_prog_sha; }; static inline const char *btf_field_type_name(enum btf_field_type type) { switch (type) { case BPF_SPIN_LOCK: return "bpf_spin_lock"; case BPF_RES_SPIN_LOCK: return "bpf_res_spin_lock"; case BPF_TIMER: return "bpf_timer"; case BPF_WORKQUEUE: return "bpf_wq"; case BPF_KPTR_UNREF: case BPF_KPTR_REF: return "kptr"; case BPF_KPTR_PERCPU: return "percpu_kptr"; case BPF_UPTR: return "uptr"; case BPF_LIST_HEAD: return "bpf_list_head"; case BPF_LIST_NODE: return "bpf_list_node"; case BPF_RB_ROOT: return "bpf_rb_root"; case BPF_RB_NODE: return "bpf_rb_node"; case BPF_REFCOUNT: return "bpf_refcount"; case BPF_TASK_WORK: return "bpf_task_work"; default: WARN_ON_ONCE(1); return "unknown"; } } #if IS_ENABLED(CONFIG_DEBUG_KERNEL) #define BPF_WARN_ONCE(cond, format...) WARN_ONCE(cond, format) #else #define BPF_WARN_ONCE(cond, format...) BUILD_BUG_ON_INVALID(cond) #endif static inline u32 btf_field_type_size(enum btf_field_type type) { switch (type) { case BPF_SPIN_LOCK: return sizeof(struct bpf_spin_lock); case BPF_RES_SPIN_LOCK: return sizeof(struct bpf_res_spin_lock); case BPF_TIMER: return sizeof(struct bpf_timer); case BPF_WORKQUEUE: return sizeof(struct bpf_wq); case BPF_KPTR_UNREF: case BPF_KPTR_REF: case BPF_KPTR_PERCPU: case BPF_UPTR: return sizeof(u64); case BPF_LIST_HEAD: return sizeof(struct bpf_list_head); case BPF_LIST_NODE: return sizeof(struct bpf_list_node); case BPF_RB_ROOT: return sizeof(struct bpf_rb_root); case BPF_RB_NODE: return sizeof(struct bpf_rb_node); case BPF_REFCOUNT: return sizeof(struct bpf_refcount); case BPF_TASK_WORK: return sizeof(struct bpf_task_work); default: WARN_ON_ONCE(1); return 0; } } static inline u32 btf_field_type_align(enum btf_field_type type) { switch (type) { case BPF_SPIN_LOCK: return __alignof__(struct bpf_spin_lock); case BPF_RES_SPIN_LOCK: return __alignof__(struct bpf_res_spin_lock); case BPF_TIMER: return __alignof__(struct bpf_timer); case BPF_WORKQUEUE: return __alignof__(struct bpf_wq); case BPF_KPTR_UNREF: case BPF_KPTR_REF: case BPF_KPTR_PERCPU: case BPF_UPTR: return __alignof__(u64); case BPF_LIST_HEAD: return __alignof__(struct bpf_list_head); case BPF_LIST_NODE: return __alignof__(struct bpf_list_node); case BPF_RB_ROOT: return __alignof__(struct bpf_rb_root); case BPF_RB_NODE: return __alignof__(struct bpf_rb_node); case BPF_REFCOUNT: return __alignof__(struct bpf_refcount); case BPF_TASK_WORK: return __alignof__(struct bpf_task_work); default: WARN_ON_ONCE(1); return 0; } } static inline void bpf_obj_init_field(const struct btf_field *field, void *addr) { memset(addr, 0, field->size); switch (field->type) { case BPF_REFCOUNT: refcount_set((refcount_t *)addr, 1); break; case BPF_RB_NODE: RB_CLEAR_NODE((struct rb_node *)addr); break; case BPF_LIST_HEAD: case BPF_LIST_NODE: INIT_LIST_HEAD((struct list_head *)addr); break; case BPF_RB_ROOT: /* RB_ROOT_CACHED 0-inits, no need to do anything after memset */ case BPF_SPIN_LOCK: case BPF_RES_SPIN_LOCK: case BPF_TIMER: case BPF_WORKQUEUE: case BPF_KPTR_UNREF: case BPF_KPTR_REF: case BPF_KPTR_PERCPU: case BPF_UPTR: case BPF_TASK_WORK: break; default: WARN_ON_ONCE(1); return; } } static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_field_type type) { if (IS_ERR_OR_NULL(rec)) return false; return rec->field_mask & type; } static inline void bpf_obj_init(const struct btf_record *rec, void *obj) { int i; if (IS_ERR_OR_NULL(rec)) return; for (i = 0; i < rec->cnt; i++) bpf_obj_init_field(&rec->fields[i], obj + rec->fields[i].offset); } /* 'dst' must be a temporary buffer and should not point to memory that is being * used in parallel by a bpf program or bpf syscall, otherwise the access from * the bpf program or bpf syscall may be corrupted by the reinitialization, * leading to weird problems. Even 'dst' is newly-allocated from bpf memory * allocator, it is still possible for 'dst' to be used in parallel by a bpf * program or bpf syscall. */ static inline void check_and_init_map_value(struct bpf_map *map, void *dst) { bpf_obj_init(map->record, dst); } /* memcpy that is used with 8-byte aligned pointers, power-of-8 size and * forced to use 'long' read/writes to try to atomically copy long counters. * Best-effort only. No barriers here, since it _will_ race with concurrent * updates from BPF programs. Called from bpf syscall and mostly used with * size 8 or 16 bytes, so ask compiler to inline it. */ static inline void bpf_long_memcpy(void *dst, const void *src, u32 size) { const long *lsrc = src; long *ldst = dst; size /= sizeof(long); while (size--) data_race(*ldst++ = *lsrc++); } /* copy everything but bpf_spin_lock, bpf_timer, and kptrs. There could be one of each. */ static inline void bpf_obj_memcpy(struct btf_record *rec, void *dst, void *src, u32 size, bool long_memcpy) { u32 curr_off = 0; int i; if (IS_ERR_OR_NULL(rec)) { if (long_memcpy) bpf_long_memcpy(dst, src, round_up(size, 8)); else memcpy(dst, src, size); return; } for (i = 0; i < rec->cnt; i++) { u32 next_off = rec->fields[i].offset; u32 sz = next_off - curr_off; memcpy(dst + curr_off, src + curr_off, sz); curr_off += rec->fields[i].size + sz; } memcpy(dst + curr_off, src + curr_off, size - curr_off); } static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) { bpf_obj_memcpy(map->record, dst, src, map->value_size, false); } static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) { bpf_obj_memcpy(map->record, dst, src, map->value_size, true); } static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) { unsigned long *src_uptr, *dst_uptr; const struct btf_field *field; int i; if (!btf_record_has_field(rec, BPF_UPTR)) return; for (i = 0, field = rec->fields; i < rec->cnt; i++, field++) { if (field->type != BPF_UPTR) continue; src_uptr = src + field->offset; dst_uptr = dst + field->offset; swap(*src_uptr, *dst_uptr); } } static inline void bpf_obj_memzero(struct btf_record *rec, void *dst, u32 size) { u32 curr_off = 0; int i; if (IS_ERR_OR_NULL(rec)) { memset(dst, 0, size); return; } for (i = 0; i < rec->cnt; i++) { u32 next_off = rec->fields[i].offset; u32 sz = next_off - curr_off; memset(dst + curr_off, 0, sz); curr_off += rec->fields[i].size + sz; } memset(dst + curr_off, 0, size - curr_off); } static inline void zero_map_value(struct bpf_map *map, void *dst) { bpf_obj_memzero(map->record, dst, map->value_size); } void copy_map_value_locked(struct bpf_map *map, void *dst, void *src, bool lock_src); void bpf_timer_cancel_and_free(void *timer); void bpf_wq_cancel_and_free(void *timer); void bpf_task_work_cancel_and_free(void *timer); void bpf_list_head_free(const struct btf_field *field, void *list_head, struct bpf_spin_lock *spin_lock); void bpf_rb_root_free(const struct btf_field *field, void *rb_root, struct bpf_spin_lock *spin_lock); u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena); u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena); int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size); struct bpf_offload_dev; struct bpf_offloaded_map; struct bpf_map_dev_ops { int (*map_get_next_key)(struct bpf_offloaded_map *map, void *key, void *next_key); int (*map_lookup_elem)(struct bpf_offloaded_map *map, void *key, void *value); int (*map_update_elem)(struct bpf_offloaded_map *map, void *key, void *value, u64 flags); int (*map_delete_elem)(struct bpf_offloaded_map *map, void *key); }; struct bpf_offloaded_map { struct bpf_map map; struct net_device *netdev; const struct bpf_map_dev_ops *dev_ops; void *dev_priv; struct list_head offloads; }; static inline struct bpf_offloaded_map *map_to_offmap(struct bpf_map *map) { return container_of(map, struct bpf_offloaded_map, map); } static inline bool bpf_map_offload_neutral(const struct bpf_map *map) { return map->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; } static inline bool bpf_map_support_seq_show(const struct bpf_map *map) { return (map->btf_value_type_id || map->btf_vmlinux_value_type_id) && map->ops->map_seq_show_elem; } int map_check_no_btf(const struct bpf_map *map, const struct btf *btf, const struct btf_type *key_type, const struct btf_type *value_type); bool bpf_map_meta_equal(const struct bpf_map *meta0, const struct bpf_map *meta1); extern const struct bpf_map_ops bpf_map_offload_ops; /* bpf_type_flag contains a set of flags that are applicable to the values of * arg_type, ret_type and reg_type. For example, a pointer value may be null, * or a memory is read-only. We classify types into two categories: base types * and extended types. Extended types are base types combined with a type flag. * * Currently there are no more than 32 base types in arg_type, ret_type and * reg_types. */ #define BPF_BASE_TYPE_BITS 8 enum bpf_type_flag { /* PTR may be NULL. */ PTR_MAYBE_NULL = BIT(0 + BPF_BASE_TYPE_BITS), /* MEM is read-only. When applied on bpf_arg, it indicates the arg is * compatible with both mutable and immutable memory. */ MEM_RDONLY = BIT(1 + BPF_BASE_TYPE_BITS), /* MEM points to BPF ring buffer reservation. */ MEM_RINGBUF = BIT(2 + BPF_BASE_TYPE_BITS), /* MEM is in user address space. */ MEM_USER = BIT(3 + BPF_BASE_TYPE_BITS), /* MEM is a percpu memory. MEM_PERCPU tags PTR_TO_BTF_ID. When tagged * with MEM_PERCPU, PTR_TO_BTF_ID _cannot_ be directly accessed. In * order to drop this tag, it must be passed into bpf_per_cpu_ptr() * or bpf_this_cpu_ptr(), which will return the pointer corresponding * to the specified cpu. */ MEM_PERCPU = BIT(4 + BPF_BASE_TYPE_BITS), /* Indicates that the argument will be released. */ OBJ_RELEASE = BIT(5 + BPF_BASE_TYPE_BITS), /* PTR is not trusted. This is only used with PTR_TO_BTF_ID, to mark * unreferenced and referenced kptr loaded from map value using a load * instruction, so that they can only be dereferenced but not escape the * BPF program into the kernel (i.e. cannot be passed as arguments to * kfunc or bpf helpers). */ PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS), /* MEM can be uninitialized. */ MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS), /* DYNPTR points to memory local to the bpf program. */ DYNPTR_TYPE_LOCAL = BIT(8 + BPF_BASE_TYPE_BITS), /* DYNPTR points to a kernel-produced ringbuf record. */ DYNPTR_TYPE_RINGBUF = BIT(9 + BPF_BASE_TYPE_BITS), /* Size is known at compile time. */ MEM_FIXED_SIZE = BIT(10 + BPF_BASE_TYPE_BITS), /* MEM is of an allocated object of type in program BTF. This is used to * tag PTR_TO_BTF_ID allocated using bpf_obj_new. */ MEM_ALLOC = BIT(11 + BPF_BASE_TYPE_BITS), /* PTR was passed from the kernel in a trusted context, and may be * passed to KF_TRUSTED_ARGS kfuncs or BPF helper functions. * Confusingly, this is _not_ the opposite of PTR_UNTRUSTED above. * PTR_UNTRUSTED refers to a kptr that was read directly from a map * without invoking bpf_kptr_xchg(). What we really need to know is * whether a pointer is safe to pass to a kfunc or BPF helper function. * While PTR_UNTRUSTED pointers are unsafe to pass to kfuncs and BPF * helpers, they do not cover all possible instances of unsafe * pointers. For example, a pointer that was obtained from walking a * struct will _not_ get the PTR_UNTRUSTED type modifier, despite the * fact that it may be NULL, invalid, etc. This is due to backwards * compatibility requirements, as this was the behavior that was first * introduced when kptrs were added. The behavior is now considered * deprecated, and PTR_UNTRUSTED will eventually be removed. * * PTR_TRUSTED, on the other hand, is a pointer that the kernel * guarantees to be valid and safe to pass to kfuncs and BPF helpers. * For example, pointers passed to tracepoint arguments are considered * PTR_TRUSTED, as are pointers that are passed to struct_ops * callbacks. As alluded to above, pointers that are obtained from * walking PTR_TRUSTED pointers are _not_ trusted. For example, if a * struct task_struct *task is PTR_TRUSTED, then accessing * task->last_wakee will lose the PTR_TRUSTED modifier when it's stored * in a BPF register. Similarly, pointers passed to certain programs * types such as kretprobes are not guaranteed to be valid, as they may * for example contain an object that was recently freed. */ PTR_TRUSTED = BIT(12 + BPF_BASE_TYPE_BITS), /* MEM is tagged with rcu and memory access needs rcu_read_lock protection. */ MEM_RCU = BIT(13 + BPF_BASE_TYPE_BITS), /* Used to tag PTR_TO_BTF_ID | MEM_ALLOC references which are non-owning. * Currently only valid for linked-list and rbtree nodes. If the nodes * have a bpf_refcount_field, they must be tagged MEM_RCU as well. */ NON_OWN_REF = BIT(14 + BPF_BASE_TYPE_BITS), /* DYNPTR points to sk_buff */ DYNPTR_TYPE_SKB = BIT(15 + BPF_BASE_TYPE_BITS), /* DYNPTR points to xdp_buff */ DYNPTR_TYPE_XDP = BIT(16 + BPF_BASE_TYPE_BITS), /* Memory must be aligned on some architectures, used in combination with * MEM_FIXED_SIZE. */ MEM_ALIGNED = BIT(17 + BPF_BASE_TYPE_BITS), /* MEM is being written to, often combined with MEM_UNINIT. Non-presence * of MEM_WRITE means that MEM is only being read. MEM_WRITE without the * MEM_UNINIT means that memory needs to be initialized since it is also * read. */ MEM_WRITE = BIT(18 + BPF_BASE_TYPE_BITS), /* DYNPTR points to skb_metadata_end()-skb_metadata_len() */ DYNPTR_TYPE_SKB_META = BIT(19 + BPF_BASE_TYPE_BITS), __BPF_TYPE_FLAG_MAX, __BPF_TYPE_LAST_FLAG = __BPF_TYPE_FLAG_MAX - 1, }; #define DYNPTR_TYPE_FLAG_MASK (DYNPTR_TYPE_LOCAL | DYNPTR_TYPE_RINGBUF | DYNPTR_TYPE_SKB \ | DYNPTR_TYPE_XDP | DYNPTR_TYPE_SKB_META) /* Max number of base types. */ #define BPF_BASE_TYPE_LIMIT (1UL << BPF_BASE_TYPE_BITS) /* Max number of all types. */ #define BPF_TYPE_LIMIT (__BPF_TYPE_LAST_FLAG | (__BPF_TYPE_LAST_FLAG - 1)) /* function argument constraints */ enum bpf_arg_type { ARG_DONTCARE = 0, /* unused argument in helper function */ /* the following constraints used to prototype * bpf_map_lookup/update/delete_elem() functions */ ARG_CONST_MAP_PTR, /* const argument used as pointer to bpf_map */ ARG_PTR_TO_MAP_KEY, /* pointer to stack used as map key */ ARG_PTR_TO_MAP_VALUE, /* pointer to stack used as map value */ /* Used to prototype bpf_memcmp() and other functions that access data * on eBPF program stack */ ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ ARG_PTR_TO_ARENA, ARG_CONST_SIZE, /* number of bytes accessed from memory */ ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ ARG_PTR_TO_CTX, /* pointer to context */ ARG_ANYTHING, /* any (initialized) argument is ok */ ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ ARG_PTR_TO_BTF_ID, /* pointer to in-kernel struct */ ARG_PTR_TO_RINGBUF_MEM, /* pointer to dynamically reserved ringbuf memory */ ARG_CONST_ALLOC_SIZE_OR_ZERO, /* number of allocated bytes requested */ ARG_PTR_TO_BTF_ID_SOCK_COMMON, /* pointer to in-kernel sock_common or bpf-mirrored bpf_sock */ ARG_PTR_TO_PERCPU_BTF_ID, /* pointer to in-kernel percpu type */ ARG_PTR_TO_FUNC, /* pointer to a bpf program function */ ARG_PTR_TO_STACK, /* pointer to stack */ ARG_PTR_TO_CONST_STR, /* pointer to a null terminated read-only string */ ARG_PTR_TO_TIMER, /* pointer to bpf_timer */ ARG_KPTR_XCHG_DEST, /* pointer to destination that kptrs are bpf_kptr_xchg'd into */ ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */ __BPF_ARG_TYPE_MAX, /* Extended arg_types. */ ARG_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MAP_VALUE, ARG_PTR_TO_MEM_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_MEM, ARG_PTR_TO_CTX_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_CTX, ARG_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_SOCKET, ARG_PTR_TO_STACK_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_STACK, ARG_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | ARG_PTR_TO_BTF_ID, /* Pointer to memory does not need to be initialized, since helper function * fills all bytes or clears them in error case. */ ARG_PTR_TO_UNINIT_MEM = MEM_UNINIT | MEM_WRITE | ARG_PTR_TO_MEM, /* Pointer to valid memory of size known at compile time. */ ARG_PTR_TO_FIXED_SIZE_MEM = MEM_FIXED_SIZE | ARG_PTR_TO_MEM, /* This must be the last entry. Its purpose is to ensure the enum is * wide enough to hold the higher bits reserved for bpf_type_flag. */ __BPF_ARG_TYPE_LIMIT = BPF_TYPE_LIMIT, }; static_assert(__BPF_ARG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); /* type of values returned from helper functions */ enum bpf_return_type { RET_INTEGER, /* function returns integer */ RET_VOID, /* function doesn't return anything */ RET_PTR_TO_MAP_VALUE, /* returns a pointer to map elem value */ RET_PTR_TO_SOCKET, /* returns a pointer to a socket */ RET_PTR_TO_TCP_SOCK, /* returns a pointer to a tcp_sock */ RET_PTR_TO_SOCK_COMMON, /* returns a pointer to a sock_common */ RET_PTR_TO_MEM, /* returns a pointer to memory */ RET_PTR_TO_MEM_OR_BTF_ID, /* returns a pointer to a valid memory or a btf_id */ RET_PTR_TO_BTF_ID, /* returns a pointer to a btf_id */ __BPF_RET_TYPE_MAX, /* Extended ret_types. */ RET_PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MAP_VALUE, RET_PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCKET, RET_PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_TCP_SOCK, RET_PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_SOCK_COMMON, RET_PTR_TO_RINGBUF_MEM_OR_NULL = PTR_MAYBE_NULL | MEM_RINGBUF | RET_PTR_TO_MEM, RET_PTR_TO_DYNPTR_MEM_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_MEM, RET_PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | RET_PTR_TO_BTF_ID, RET_PTR_TO_BTF_ID_TRUSTED = PTR_TRUSTED | RET_PTR_TO_BTF_ID, /* This must be the last entry. Its purpose is to ensure the enum is * wide enough to hold the higher bits reserved for bpf_type_flag. */ __BPF_RET_TYPE_LIMIT = BPF_TYPE_LIMIT, }; static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying */ struct bpf_func_proto { u64 (*func)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); bool gpl_only; bool pkt_access; bool might_sleep; /* set to true if helper follows contract for llvm * attribute bpf_fastcall: * - void functions do not scratch r0 * - functions taking N arguments scratch only registers r1-rN */ bool allow_fastcall; enum bpf_return_type ret_type; union { struct { enum bpf_arg_type arg1_type; enum bpf_arg_type arg2_type; enum bpf_arg_type arg3_type; enum bpf_arg_type arg4_type; enum bpf_arg_type arg5_type; }; enum bpf_arg_type arg_type[5]; }; union { struct { u32 *arg1_btf_id; u32 *arg2_btf_id; u32 *arg3_btf_id; u32 *arg4_btf_id; u32 *arg5_btf_id; }; u32 *arg_btf_id[5]; struct { size_t arg1_size; size_t arg2_size; size_t arg3_size; size_t arg4_size; size_t arg5_size; }; size_t arg_size[5]; }; int *ret_btf_id; /* return value btf_id */ bool (*allowed)(const struct bpf_prog *prog); }; /* bpf_context is intentionally undefined structure. Pointer to bpf_context is * the first argument to eBPF programs. * For socket filters: 'struct bpf_context *' == 'struct sk_buff *' */ struct bpf_context; enum bpf_access_type { BPF_READ = 1, BPF_WRITE = 2 }; /* types of values stored in eBPF registers */ /* Pointer types represent: * pointer * pointer + imm * pointer + (u16) var * pointer + (u16) var + imm * if (range > 0) then [ptr, ptr + range - off) is safe to access * if (id > 0) means that some 'var' was added * if (off > 0) means that 'imm' was added */ enum bpf_reg_type { NOT_INIT = 0, /* nothing was written into register */ SCALAR_VALUE, /* reg doesn't contain a valid pointer */ PTR_TO_CTX, /* reg points to bpf_context */ CONST_PTR_TO_MAP, /* reg points to struct bpf_map */ PTR_TO_MAP_VALUE, /* reg points to map element value */ PTR_TO_MAP_KEY, /* reg points to a map element key */ PTR_TO_STACK, /* reg == frame_pointer + offset */ PTR_TO_PACKET_META, /* skb->data - meta_len */ PTR_TO_PACKET, /* reg points to skb->data */ PTR_TO_PACKET_END, /* skb->data + headlen */ PTR_TO_FLOW_KEYS, /* reg points to bpf_flow_keys */ PTR_TO_SOCKET, /* reg points to struct bpf_sock */ PTR_TO_SOCK_COMMON, /* reg points to sock_common */ PTR_TO_TCP_SOCK, /* reg points to struct tcp_sock */ PTR_TO_TP_BUFFER, /* reg points to a writable raw tp's buffer */ PTR_TO_XDP_SOCK, /* reg points to struct xdp_sock */ /* PTR_TO_BTF_ID points to a kernel struct that does not need * to be null checked by the BPF program. This does not imply the * pointer is _not_ null and in practice this can easily be a null * pointer when reading pointer chains. The assumption is program * context will handle null pointer dereference typically via fault * handling. The verifier must keep this in mind and can make no * assumptions about null or non-null when doing branch analysis. * Further, when passed into helpers the helpers can not, without * additional context, assume the value is non-null. */ PTR_TO_BTF_ID, PTR_TO_MEM, /* reg points to valid memory region */ PTR_TO_ARENA, PTR_TO_BUF, /* reg points to a read/write buffer */ PTR_TO_FUNC, /* reg points to a bpf program function */ CONST_PTR_TO_DYNPTR, /* reg points to a const struct bpf_dynptr */ __BPF_REG_TYPE_MAX, /* Extended reg_types. */ PTR_TO_MAP_VALUE_OR_NULL = PTR_MAYBE_NULL | PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCKET, PTR_TO_SOCK_COMMON_OR_NULL = PTR_MAYBE_NULL | PTR_TO_SOCK_COMMON, PTR_TO_TCP_SOCK_OR_NULL = PTR_MAYBE_NULL | PTR_TO_TCP_SOCK, /* PTR_TO_BTF_ID_OR_NULL points to a kernel struct that has not * been checked for null. Used primarily to inform the verifier * an explicit null check is required for this struct. */ PTR_TO_BTF_ID_OR_NULL = PTR_MAYBE_NULL | PTR_TO_BTF_ID, /* This must be the last entry. Its purpose is to ensure the enum is * wide enough to hold the higher bits reserved for bpf_type_flag. */ __BPF_REG_TYPE_LIMIT = BPF_TYPE_LIMIT, }; static_assert(__BPF_REG_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); /* The information passed from prog-specific *_is_valid_access * back to the verifier. */ struct bpf_insn_access_aux { enum bpf_reg_type reg_type; bool is_ldsx; union { int ctx_field_size; struct { struct btf *btf; u32 btf_id; u32 ref_obj_id; }; }; struct bpf_verifier_log *log; /* for verbose logs */ bool is_retval; /* is accessing function return value ? */ }; static inline void bpf_ctx_record_field_size(struct bpf_insn_access_aux *aux, u32 size) { aux->ctx_field_size = size; } static bool bpf_is_ldimm64(const struct bpf_insn *insn) { return insn->code == (BPF_LD | BPF_IMM | BPF_DW); } static inline bool bpf_pseudo_func(const struct bpf_insn *insn) { return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC; } /* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an * atomic load or store, and false if it is a read-modify-write instruction. */ static inline bool bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) { switch (atomic_insn->imm) { case BPF_LOAD_ACQ: case BPF_STORE_REL: return true; default: return false; } } struct bpf_prog_ops { int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); }; struct bpf_reg_state; struct bpf_verifier_ops { /* return eBPF function prototype for verification */ const struct bpf_func_proto * (*get_func_proto)(enum bpf_func_id func_id, const struct bpf_prog *prog); /* return true if 'size' wide access at offset 'off' within bpf_context * with 'type' (read or write) is allowed */ bool (*is_valid_access)(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info); int (*gen_prologue)(struct bpf_insn *insn, bool direct_write, const struct bpf_prog *prog); int (*gen_epilogue)(struct bpf_insn *insn, const struct bpf_prog *prog, s16 ctx_stack_off); int (*gen_ld_abs)(const struct bpf_insn *orig, struct bpf_insn *insn_buf); u32 (*convert_ctx_access)(enum bpf_access_type type, const struct bpf_insn *src, struct bpf_insn *dst, struct bpf_prog *prog, u32 *target_size); int (*btf_struct_access)(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, int off, int size); }; struct bpf_prog_offload_ops { /* verifier basic callbacks */ int (*insn_hook)(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx); int (*finalize)(struct bpf_verifier_env *env); /* verifier optimization callbacks (called after .finalize) */ int (*replace_insn)(struct bpf_verifier_env *env, u32 off, struct bpf_insn *insn); int (*remove_insns)(struct bpf_verifier_env *env, u32 off, u32 cnt); /* program management callbacks */ int (*prepare)(struct bpf_prog *prog); int (*translate)(struct bpf_prog *prog); void (*destroy)(struct bpf_prog *prog); }; struct bpf_prog_offload { struct bpf_prog *prog; struct net_device *netdev; struct bpf_offload_dev *offdev; void *dev_priv; struct list_head offloads; bool dev_state; bool opt_failed; void *jited_image; u32 jited_len; }; /* The longest tracepoint has 12 args. * See include/trace/bpf_probe.h */ #define MAX_BPF_FUNC_ARGS 12 /* The maximum number of arguments passed through registers * a single function may have. */ #define MAX_BPF_FUNC_REG_ARGS 5 /* The argument is a structure or a union. */ #define BTF_FMODEL_STRUCT_ARG BIT(0) /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) struct btf_func_model { u8 ret_size; u8 ret_flags; u8 nr_args; u8 arg_size[MAX_BPF_FUNC_ARGS]; u8 arg_flags[MAX_BPF_FUNC_ARGS]; }; /* Restore arguments before returning from trampoline to let original function * continue executing. This flag is used for fentry progs when there are no * fexit progs. */ #define BPF_TRAMP_F_RESTORE_REGS BIT(0) /* Call original function after fentry progs, but before fexit progs. * Makes sense for fentry/fexit, normal calls and indirect calls. */ #define BPF_TRAMP_F_CALL_ORIG BIT(1) /* Skip current frame and return to parent. Makes sense for fentry/fexit * programs only. Should not be used with normal calls and indirect calls. */ #define BPF_TRAMP_F_SKIP_FRAME BIT(2) /* Store IP address of the caller on the trampoline stack, * so it's available for trampoline's programs. */ #define BPF_TRAMP_F_IP_ARG BIT(3) /* Return the return value of fentry prog. Only used by bpf_struct_ops. */ #define BPF_TRAMP_F_RET_FENTRY_RET BIT(4) /* Get original function from stack instead of from provided direct address. * Makes sense for trampolines with fexit or fmod_ret programs. */ #define BPF_TRAMP_F_ORIG_STACK BIT(5) /* This trampoline is on a function with another ftrace_ops with IPMODIFY, * e.g., a live patch. This flag is set and cleared by ftrace call backs, */ #define BPF_TRAMP_F_SHARE_IPMODIFY BIT(6) /* Indicate that current trampoline is in a tail call context. Then, it has to * cache and restore tail_call_cnt to avoid infinite tail call loop. */ #define BPF_TRAMP_F_TAIL_CALL_CTX BIT(7) /* * Indicate the trampoline should be suitable to receive indirect calls; * without this indirectly calling the generated code can result in #UD/#CP, * depending on the CFI options. * * Used by bpf_struct_ops. * * Incompatible with FENTRY usage, overloads @func_addr argument. */ #define BPF_TRAMP_F_INDIRECT BIT(8) /* Each call __bpf_prog_enter + call bpf_func + call __bpf_prog_exit is ~50 * bytes on x86. */ enum { #if defined(__s390x__) BPF_MAX_TRAMP_LINKS = 27, #else BPF_MAX_TRAMP_LINKS = 38, #endif }; struct bpf_tramp_links { struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS]; int nr_links; }; struct bpf_tramp_run_ctx; /* Different use cases for BPF trampoline: * 1. replace nop at the function entry (kprobe equivalent) * flags = BPF_TRAMP_F_RESTORE_REGS * fentry = a set of programs to run before returning from trampoline * * 2. replace nop at the function entry (kprobe + kretprobe equivalent) * flags = BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_SKIP_FRAME * orig_call = fentry_ip + MCOUNT_INSN_SIZE * fentry = a set of program to run before calling original function * fexit = a set of program to run after original function * * 3. replace direct call instruction anywhere in the function body * or assign a function pointer for indirect call (like tcp_congestion_ops->cong_avoid) * With flags = 0 * fentry = a set of programs to run before returning from trampoline * With flags = BPF_TRAMP_F_CALL_ORIG * orig_call = original callback addr or direct function addr * fentry = a set of program to run before calling original function * fexit = a set of program to run after original function */ struct bpf_tramp_image; int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, void *func_addr); void *arch_alloc_bpf_trampoline(unsigned int size); void arch_free_bpf_trampoline(void *image, unsigned int size); int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size); int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, struct bpf_tramp_links *tlinks, void *func_addr); u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_prog_exit_sleepable_recur(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx); void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr); void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr); typedef u64 (*bpf_trampoline_enter_t)(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); typedef void (*bpf_trampoline_exit_t)(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx); bpf_trampoline_enter_t bpf_trampoline_enter(const struct bpf_prog *prog); bpf_trampoline_exit_t bpf_trampoline_exit(const struct bpf_prog *prog); struct bpf_ksym { unsigned long start; unsigned long end; char name[KSYM_NAME_LEN]; struct list_head lnode; struct latch_tree_node tnode; bool prog; }; enum bpf_tramp_prog_type { BPF_TRAMP_FENTRY, BPF_TRAMP_FEXIT, BPF_TRAMP_MODIFY_RETURN, BPF_TRAMP_MAX, BPF_TRAMP_REPLACE, /* more than MAX */ }; struct bpf_tramp_image { void *image; int size; struct bpf_ksym ksym; struct percpu_ref pcref; void *ip_after_call; void *ip_epilogue; union { struct rcu_head rcu; struct work_struct work; }; }; struct bpf_trampoline { /* hlist for trampoline_table */ struct hlist_node hlist; struct ftrace_ops *fops; /* serializes access to fields of this trampoline */ struct mutex mutex; refcount_t refcnt; u32 flags; u64 key; struct { struct btf_func_model model; void *addr; bool ftrace_managed; } func; /* if !NULL this is BPF_PROG_TYPE_EXT program that extends another BPF * program by replacing one of its functions. func.addr is the address * of the function it replaced. */ struct bpf_prog *extension_prog; /* list of BPF programs using this trampoline */ struct hlist_head progs_hlist[BPF_TRAMP_MAX]; /* Number of attached programs. A counter per kind. */ int progs_cnt[BPF_TRAMP_MAX]; /* Executable image of trampoline */ struct bpf_tramp_image *cur_image; }; struct bpf_attach_target_info { struct btf_func_model fmodel; long tgt_addr; struct module *tgt_mod; const char *tgt_name; const struct btf_type *tgt_type; }; #define BPF_DISPATCHER_MAX 48 /* Fits in 2048B */ struct bpf_dispatcher_prog { struct bpf_prog *prog; refcount_t users; }; struct bpf_dispatcher { /* dispatcher mutex */ struct mutex mutex; void *func; struct bpf_dispatcher_prog progs[BPF_DISPATCHER_MAX]; int num_progs; void *image; void *rw_image; u32 image_off; struct bpf_ksym ksym; #ifdef CONFIG_HAVE_STATIC_CALL struct static_call_key *sc_key; void *sc_tramp; #endif }; #ifndef __bpfcall #define __bpfcall __nocfi #endif static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func( const void *ctx, const struct bpf_insn *insnsi, bpf_func_t bpf_func) { return bpf_func(ctx, insnsi); } /* the implementation of the opaque uapi struct bpf_dynptr */ struct bpf_dynptr_kern { void *data; /* Size represents the number of usable bytes of dynptr data. * If for example the offset is at 4 for a local dynptr whose data is * of type u64, the number of usable bytes is 4. * * The upper 8 bits are reserved. It is as follows: * Bits 0 - 23 = size * Bits 24 - 30 = dynptr type * Bit 31 = whether dynptr is read-only */ u32 size; u32 offset; } __aligned(8); enum bpf_dynptr_type { BPF_DYNPTR_TYPE_INVALID, /* Points to memory that is local to the bpf program */ BPF_DYNPTR_TYPE_LOCAL, /* Underlying data is a ringbuf record */ BPF_DYNPTR_TYPE_RINGBUF, /* Underlying data is a sk_buff */ BPF_DYNPTR_TYPE_SKB, /* Underlying data is a xdp_buff */ BPF_DYNPTR_TYPE_XDP, /* Points to skb_metadata_end()-skb_metadata_len() */ BPF_DYNPTR_TYPE_SKB_META, }; int bpf_dynptr_check_size(u32 size); u32 __bpf_dynptr_size(const struct bpf_dynptr_kern *ptr); const void *__bpf_dynptr_data(const struct bpf_dynptr_kern *ptr, u32 len); void *__bpf_dynptr_data_rw(const struct bpf_dynptr_kern *ptr, u32 len); bool __bpf_dynptr_is_rdonly(const struct bpf_dynptr_kern *ptr); int __bpf_dynptr_write(const struct bpf_dynptr_kern *dst, u32 offset, void *src, u32 len, u64 flags); void *bpf_dynptr_slice_rdwr(const struct bpf_dynptr *p, u32 offset, void *buffer__opt, u32 buffer__szk); static inline int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u32 offset, u32 len) { u32 size = __bpf_dynptr_size(ptr); if (len > size || offset > size - len) return -E2BIG; return 0; } #ifdef CONFIG_BPF_JIT int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog); int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog); struct bpf_trampoline *bpf_trampoline_get(u64 key, struct bpf_attach_target_info *tgt_info); void bpf_trampoline_put(struct bpf_trampoline *tr); int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs); /* * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn * indirection with a direct call to the bpf program. If the architecture does * not have STATIC_CALL, avoid a double-indirection. */ #ifdef CONFIG_HAVE_STATIC_CALL #define __BPF_DISPATCHER_SC_INIT(_name) \ .sc_key = &STATIC_CALL_KEY(_name), \ .sc_tramp = STATIC_CALL_TRAMP_ADDR(_name), #define __BPF_DISPATCHER_SC(name) \ DEFINE_STATIC_CALL(bpf_dispatcher_##name##_call, bpf_dispatcher_nop_func) #define __BPF_DISPATCHER_CALL(name) \ static_call(bpf_dispatcher_##name##_call)(ctx, insnsi, bpf_func) #define __BPF_DISPATCHER_UPDATE(_d, _new) \ __static_call_update((_d)->sc_key, (_d)->sc_tramp, (_new)) #else #define __BPF_DISPATCHER_SC_INIT(name) #define __BPF_DISPATCHER_SC(name) #define __BPF_DISPATCHER_CALL(name) bpf_func(ctx, insnsi) #define __BPF_DISPATCHER_UPDATE(_d, _new) #endif #define BPF_DISPATCHER_INIT(_name) { \ .mutex = __MUTEX_INITIALIZER(_name.mutex), \ .func = &_name##_func, \ .progs = {}, \ .num_progs = 0, \ .image = NULL, \ .image_off = 0, \ .ksym = { \ .name = #_name, \ .lnode = LIST_HEAD_INIT(_name.ksym.lnode), \ }, \ __BPF_DISPATCHER_SC_INIT(_name##_call) \ } #define DEFINE_BPF_DISPATCHER(name) \ __BPF_DISPATCHER_SC(name); \ noinline __bpfcall unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ const struct bpf_insn *insnsi, \ bpf_func_t bpf_func) \ { \ return __BPF_DISPATCHER_CALL(name); \ } \ EXPORT_SYMBOL(bpf_dispatcher_##name##_func); \ struct bpf_dispatcher bpf_dispatcher_##name = \ BPF_DISPATCHER_INIT(bpf_dispatcher_##name); #define DECLARE_BPF_DISPATCHER(name) \ unsigned int bpf_dispatcher_##name##_func( \ const void *ctx, \ const struct bpf_insn *insnsi, \ bpf_func_t bpf_func); \ extern struct bpf_dispatcher bpf_dispatcher_##name; #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_##name##_func #define BPF_DISPATCHER_PTR(name) (&bpf_dispatcher_##name) void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, struct bpf_prog *to); /* Called only from JIT-enabled code, so there's no need for stubs. */ void bpf_image_ksym_init(void *data, unsigned int size, struct bpf_ksym *ksym); void bpf_image_ksym_add(struct bpf_ksym *ksym); void bpf_image_ksym_del(struct bpf_ksym *ksym); void bpf_ksym_add(struct bpf_ksym *ksym); void bpf_ksym_del(struct bpf_ksym *ksym); int bpf_jit_charge_modmem(u32 size); void bpf_jit_uncharge_modmem(u32 size); bool bpf_prog_has_trampoline(const struct bpf_prog *prog); #else static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog) { return -ENOTSUPP; } static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog) { return -ENOTSUPP; } static inline struct bpf_trampoline *bpf_trampoline_get(u64 key, struct bpf_attach_target_info *tgt_info) { return NULL; } static inline void bpf_trampoline_put(struct bpf_trampoline *tr) {} #define DEFINE_BPF_DISPATCHER(name) #define DECLARE_BPF_DISPATCHER(name) #define BPF_DISPATCHER_FUNC(name) bpf_dispatcher_nop_func #define BPF_DISPATCHER_PTR(name) NULL static inline void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, struct bpf_prog *to) {} static inline bool is_bpf_image_address(unsigned long address) { return false; } static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog) { return false; } #endif struct bpf_func_info_aux { u16 linkage; bool unreliable; bool called : 1; bool verified : 1; }; enum bpf_jit_poke_reason { BPF_POKE_REASON_TAIL_CALL, }; /* Descriptor of pokes pointing /into/ the JITed image. */ struct bpf_jit_poke_descriptor { void *tailcall_target; void *tailcall_bypass; void *bypass_addr; void *aux; union { struct { struct bpf_map *map; u32 key; } tail_call; }; bool tailcall_target_stable; u8 adj_off; u16 reason; u32 insn_idx; }; /* reg_type info for ctx arguments */ struct bpf_ctx_arg_aux { u32 offset; enum bpf_reg_type reg_type; struct btf *btf; u32 btf_id; u32 ref_obj_id; bool refcounted; }; struct btf_mod_pair { struct btf *btf; struct module *module; }; struct bpf_kfunc_desc_tab; enum bpf_stream_id { BPF_STDOUT = 1, BPF_STDERR = 2, }; struct bpf_stream_elem { struct llist_node node; int total_len; int consumed_len; char str[]; }; enum { /* 100k bytes */ BPF_STREAM_MAX_CAPACITY = 100000ULL, }; struct bpf_stream { atomic_t capacity; struct llist_head log; /* list of in-flight stream elements in LIFO order */ struct mutex lock; /* lock protecting backlog_{head,tail} */ struct llist_node *backlog_head; /* list of in-flight stream elements in FIFO order */ struct llist_node *backlog_tail; /* tail of the list above */ }; struct bpf_stream_stage { struct llist_head log; int len; }; struct bpf_prog_aux { atomic64_t refcnt; u32 used_map_cnt; u32 used_btf_cnt; u32 max_ctx_offset; u32 max_pkt_offset; u32 max_tp_access; u32 stack_depth; u32 id; u32 func_cnt; /* used by non-func prog as the number of func progs */ u32 real_func_cnt; /* includes hidden progs, only used for JIT and freeing progs */ u32 func_idx; /* 0 for non-func prog, the index in func array for func prog */ u32 attach_btf_id; /* in-kernel BTF type id to attach to */ u32 attach_st_ops_member_off; u32 ctx_arg_info_size; u32 max_rdonly_access; u32 max_rdwr_access; struct btf *attach_btf; struct bpf_ctx_arg_aux *ctx_arg_info; void __percpu *priv_stack_ptr; struct mutex dst_mutex; /* protects dst_* pointers below, *after* prog becomes visible */ struct bpf_prog *dst_prog; struct bpf_trampoline *dst_trampoline; enum bpf_prog_type saved_dst_prog_type; enum bpf_attach_type saved_dst_attach_type; bool verifier_zext; /* Zero extensions has been inserted by verifier. */ bool dev_bound; /* Program is bound to the netdev. */ bool offload_requested; /* Program is bound and offloaded to the netdev. */ bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ bool attach_tracing_prog; /* true if tracing another tracing program */ bool func_proto_unreliable; bool tail_call_reachable; bool xdp_has_frags; bool exception_cb; bool exception_boundary; bool is_extended; /* true if extended by freplace program */ bool jits_use_priv_stack; bool priv_stack_requested; bool changes_pkt_data; bool might_sleep; bool kprobe_write_ctx; u64 prog_array_member_cnt; /* counts how many times as member of prog_array */ struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */ struct bpf_arena *arena; void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */ /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ const struct btf_type *attach_func_proto; /* function name for valid attach_btf_id */ const char *attach_func_name; struct bpf_prog **func; struct bpf_prog_aux *main_prog_aux; void *jit_data; /* JIT specific data. arch dependent */ struct bpf_jit_poke_descriptor *poke_tab; struct bpf_kfunc_desc_tab *kfunc_tab; struct bpf_kfunc_btf_tab *kfunc_btf_tab; u32 size_poke_tab; #ifdef CONFIG_FINEIBT struct bpf_ksym ksym_prefix; #endif struct bpf_ksym ksym; const struct bpf_prog_ops *ops; const struct bpf_struct_ops *st_ops; struct bpf_map **used_maps; struct mutex used_maps_mutex; /* mutex for used_maps and used_map_cnt */ struct btf_mod_pair *used_btfs; struct bpf_prog *prog; struct user_struct *user; u64 load_time; /* ns since boottime */ u32 verified_insns; int cgroup_atype; /* enum cgroup_bpf_attach_type */ struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; char name[BPF_OBJ_NAME_LEN]; u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64); #ifdef CONFIG_SECURITY void *security; #endif struct bpf_token *token; struct bpf_prog_offload *offload; struct btf *btf; struct bpf_func_info *func_info; struct bpf_func_info_aux *func_info_aux; /* bpf_line_info loaded from userspace. linfo->insn_off * has the xlated insn offset. * Both the main and sub prog share the same linfo. * The subprog can access its first linfo by * using the linfo_idx. */ struct bpf_line_info *linfo; /* jited_linfo is the jited addr of the linfo. It has a * one to one mapping to linfo: * jited_linfo[i] is the jited addr for the linfo[i]->insn_off. * Both the main and sub prog share the same jited_linfo. * The subprog can access its first jited_linfo by * using the linfo_idx. */ void **jited_linfo; u32 func_info_cnt; u32 nr_linfo; /* subprog can use linfo_idx to access its first linfo and * jited_linfo. * main prog always has linfo_idx == 0 */ u32 linfo_idx; struct module *mod; u32 num_exentries; struct exception_table_entry *extable; union { struct work_struct work; struct rcu_head rcu; }; struct bpf_stream stream[2]; }; struct bpf_prog { u16 pages; /* Number of allocated pages */ u16 jited:1, /* Is our filter JIT'ed? */ jit_requested:1,/* archs need to JIT the prog */ gpl_compatible:1, /* Is filter GPL compatible? */ cb_access:1, /* Is control block accessed? */ dst_needed:1, /* Do we need dst entry? */ blinding_requested:1, /* needs constant blinding */ blinded:1, /* Was blinded */ is_func:1, /* program is a bpf function */ kprobe_override:1, /* Do we override a kprobe? */ has_callchain_buf:1, /* callchain buffer allocated? */ enforce_expected_attach_type:1, /* Enforce expected_attach_type checking at attach time */ call_get_stack:1, /* Do we call bpf_get_stack() or bpf_get_stackid() */ call_get_func_ip:1, /* Do we call get_func_ip() */ tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */ sleepable:1; /* BPF program is sleepable */ enum bpf_prog_type type; /* Type of BPF program */ enum bpf_attach_type expected_attach_type; /* For some prog types */ u32 len; /* Number of filter blocks */ u32 jited_len; /* Size of jited insns in bytes */ union { u8 digest[SHA256_DIGEST_SIZE]; u8 tag[BPF_TAG_SIZE]; }; struct bpf_prog_stats __percpu *stats; int __percpu *active; unsigned int (*bpf_func)(const void *ctx, const struct bpf_insn *insn); struct bpf_prog_aux *aux; /* Auxiliary fields */ struct sock_fprog_kern *orig_prog; /* Original BPF program */ /* Instructions for interpreter */ union { DECLARE_FLEX_ARRAY(struct sock_filter, insns); DECLARE_FLEX_ARRAY(struct bpf_insn, insnsi); }; }; struct bpf_array_aux { /* Programs with direct jumps into programs part of this array. */ struct list_head poke_progs; struct bpf_map *map; struct mutex poke_mutex; struct work_struct work; }; struct bpf_link { atomic64_t refcnt; u32 id; enum bpf_link_type type; const struct bpf_link_ops *ops; struct bpf_prog *prog; u32 flags; enum bpf_attach_type attach_type; /* rcu is used before freeing, work can be used to schedule that * RCU-based freeing before that, so they never overlap */ union { struct rcu_head rcu; struct work_struct work; }; /* whether BPF link itself has "sleepable" semantics, which can differ * from underlying BPF program having a "sleepable" semantics, as BPF * link's semantics is determined by target attach hook */ bool sleepable; }; struct bpf_link_ops { void (*release)(struct bpf_link *link); /* deallocate link resources callback, called without RCU grace period * waiting */ void (*dealloc)(struct bpf_link *link); /* deallocate link resources callback, called after RCU grace period; * if either the underlying BPF program is sleepable or BPF link's * target hook is sleepable, we'll go through tasks trace RCU GP and * then "classic" RCU GP; this need for chaining tasks trace and * classic RCU GPs is designated by setting bpf_link->sleepable flag */ void (*dealloc_deferred)(struct bpf_link *link); int (*detach)(struct bpf_link *link); int (*update_prog)(struct bpf_link *link, struct bpf_prog *new_prog, struct bpf_prog *old_prog); void (*show_fdinfo)(const struct bpf_link *link, struct seq_file *seq); int (*fill_link_info)(const struct bpf_link *link, struct bpf_link_info *info); int (*update_map)(struct bpf_link *link, struct bpf_map *new_map, struct bpf_map *old_map); __poll_t (*poll)(struct file *file, struct poll_table_struct *pts); }; struct bpf_tramp_link { struct bpf_link link; struct hlist_node tramp_hlist; u64 cookie; }; struct bpf_shim_tramp_link { struct bpf_tramp_link link; struct bpf_trampoline *trampoline; }; struct bpf_tracing_link { struct bpf_tramp_link link; struct bpf_trampoline *trampoline; struct bpf_prog *tgt_prog; }; struct bpf_raw_tp_link { struct bpf_link link; struct bpf_raw_event_map *btp; u64 cookie; }; struct bpf_link_primer { struct bpf_link *link; struct file *file; int fd; u32 id; }; struct bpf_mount_opts { kuid_t uid; kgid_t gid; umode_t mode; /* BPF token-related delegation options */ u64 delegate_cmds; u64 delegate_maps; u64 delegate_progs; u64 delegate_attachs; }; struct bpf_token { struct work_struct work; atomic64_t refcnt; struct user_namespace *userns; u64 allowed_cmds; u64 allowed_maps; u64 allowed_progs; u64 allowed_attachs; #ifdef CONFIG_SECURITY void *security; #endif }; struct bpf_struct_ops_value; struct btf_member; #define BPF_STRUCT_OPS_MAX_NR_MEMBERS 64 /** * struct bpf_struct_ops - A structure of callbacks allowing a subsystem to * define a BPF_MAP_TYPE_STRUCT_OPS map type composed * of BPF_PROG_TYPE_STRUCT_OPS progs. * @verifier_ops: A structure of callbacks that are invoked by the verifier * when determining whether the struct_ops progs in the * struct_ops map are valid. * @init: A callback that is invoked a single time, and before any other * callback, to initialize the structure. A nonzero return value means * the subsystem could not be initialized. * @check_member: When defined, a callback invoked by the verifier to allow * the subsystem to determine if an entry in the struct_ops map * is valid. A nonzero return value means that the map is * invalid and should be rejected by the verifier. * @init_member: A callback that is invoked for each member of the struct_ops * map to allow the subsystem to initialize the member. A nonzero * value means the member could not be initialized. This callback * is exclusive with the @type, @type_id, @value_type, and * @value_id fields. * @reg: A callback that is invoked when the struct_ops map has been * initialized and is being attached to. Zero means the struct_ops map * has been successfully registered and is live. A nonzero return value * means the struct_ops map could not be registered. * @unreg: A callback that is invoked when the struct_ops map should be * unregistered. * @update: A callback that is invoked when the live struct_ops map is being * updated to contain new values. This callback is only invoked when * the struct_ops map is loaded with BPF_F_LINK. If not defined, the * it is assumed that the struct_ops map cannot be updated. * @validate: A callback that is invoked after all of the members have been * initialized. This callback should perform static checks on the * map, meaning that it should either fail or succeed * deterministically. A struct_ops map that has been validated may * not necessarily succeed in being registered if the call to @reg * fails. For example, a valid struct_ops map may be loaded, but * then fail to be registered due to there being another active * struct_ops map on the system in the subsystem already. For this * reason, if this callback is not defined, the check is skipped as * the struct_ops map will have final verification performed in * @reg. * @type: BTF type. * @value_type: Value type. * @name: The name of the struct bpf_struct_ops object. * @func_models: Func models * @type_id: BTF type id. * @value_id: BTF value id. */ struct bpf_struct_ops { const struct bpf_verifier_ops *verifier_ops; int (*init)(struct btf *btf); int (*check_member)(const struct btf_type *t, const struct btf_member *member, const struct bpf_prog *prog); int (*init_member)(const struct btf_type *t, const struct btf_member *member, void *kdata, const void *udata); int (*reg)(void *kdata, struct bpf_link *link); void (*unreg)(void *kdata, struct bpf_link *link); int (*update)(void *kdata, void *old_kdata, struct bpf_link *link); int (*validate)(void *kdata); void *cfi_stubs; struct module *owner; const char *name; struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; }; /* Every member of a struct_ops type has an instance even a member is not * an operator (function pointer). The "info" field will be assigned to * prog->aux->ctx_arg_info of BPF struct_ops programs to provide the * argument information required by the verifier to verify the program. * * btf_ctx_access() will lookup prog->aux->ctx_arg_info to find the * corresponding entry for an given argument. */ struct bpf_struct_ops_arg_info { struct bpf_ctx_arg_aux *info; u32 cnt; }; struct bpf_struct_ops_desc { struct bpf_struct_ops *st_ops; const struct btf_type *type; const struct btf_type *value_type; u32 type_id; u32 value_id; /* Collection of argument information for each member */ struct bpf_struct_ops_arg_info *arg_info; }; enum bpf_struct_ops_state { BPF_STRUCT_OPS_STATE_INIT, BPF_STRUCT_OPS_STATE_INUSE, BPF_STRUCT_OPS_STATE_TOBEFREE, BPF_STRUCT_OPS_STATE_READY, }; struct bpf_struct_ops_common_value { refcount_t refcnt; enum bpf_struct_ops_state state; }; #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) /* This macro helps developer to register a struct_ops type and generate * type information correctly. Developers should use this macro to register * a struct_ops type instead of calling __register_bpf_struct_ops() directly. */ #define register_bpf_struct_ops(st_ops, type) \ ({ \ struct bpf_struct_ops_##type { \ struct bpf_struct_ops_common_value common; \ struct type data ____cacheline_aligned_in_smp; \ }; \ BTF_TYPE_EMIT(struct bpf_struct_ops_##type); \ __register_bpf_struct_ops(st_ops); \ }) #define BPF_MODULE_OWNER ((void *)((0xeB9FUL << 2) + POISON_POINTER_DELTA)) bool bpf_struct_ops_get(const void *kdata); void bpf_struct_ops_put(const void *kdata); int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff); int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, void *value); int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, struct bpf_tramp_link *link, const struct btf_func_model *model, void *stub_func, void **image, u32 *image_off, bool allow_alloc); void bpf_struct_ops_image_free(void *image); static inline bool bpf_try_module_get(const void *data, struct module *owner) { if (owner == BPF_MODULE_OWNER) return bpf_struct_ops_get(data); else return try_module_get(owner); } static inline void bpf_module_put(const void *data, struct module *owner) { if (owner == BPF_MODULE_OWNER) bpf_struct_ops_put(data); else module_put(owner); } int bpf_struct_ops_link_create(union bpf_attr *attr); u32 bpf_struct_ops_id(const void *kdata); #ifdef CONFIG_NET /* Define it here to avoid the use of forward declaration */ struct bpf_dummy_ops_state { int val; }; struct bpf_dummy_ops { int (*test_1)(struct bpf_dummy_ops_state *cb); int (*test_2)(struct bpf_dummy_ops_state *cb, int a1, unsigned short a2, char a3, unsigned long a4); int (*test_sleepable)(struct bpf_dummy_ops_state *cb); }; int bpf_struct_ops_test_run(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); #endif int bpf_struct_ops_desc_init(struct bpf_struct_ops_desc *st_ops_desc, struct btf *btf, struct bpf_verifier_log *log); void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map); void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc); #else #define register_bpf_struct_ops(st_ops, type) ({ (void *)(st_ops); 0; }) static inline bool bpf_try_module_get(const void *data, struct module *owner) { return try_module_get(owner); } static inline void bpf_module_put(const void *data, struct module *owner) { module_put(owner); } static inline int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff) { return -ENOTSUPP; } static inline int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, void *value) { return -EINVAL; } static inline int bpf_struct_ops_link_create(union bpf_attr *attr) { return -EOPNOTSUPP; } static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struct bpf_map *map) { } static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc) { } #endif int bpf_prog_ctx_arg_info_init(struct bpf_prog *prog, const struct bpf_ctx_arg_aux *info, u32 cnt); #if defined(CONFIG_CGROUP_BPF) && defined(CONFIG_BPF_LSM) int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, int cgroup_atype, enum bpf_attach_type attach_type); void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog); #else static inline int bpf_trampoline_link_cgroup_shim(struct bpf_prog *prog, int cgroup_atype, enum bpf_attach_type attach_type) { return -EOPNOTSUPP; } static inline void bpf_trampoline_unlink_cgroup_shim(struct bpf_prog *prog) { } #endif struct bpf_array { struct bpf_map map; u32 elem_size; u32 index_mask; struct bpf_array_aux *aux; union { DECLARE_FLEX_ARRAY(char, value) __aligned(8); DECLARE_FLEX_ARRAY(void *, ptrs) __aligned(8); DECLARE_FLEX_ARRAY(void __percpu *, pptrs) __aligned(8); }; }; #define BPF_COMPLEXITY_LIMIT_INSNS 1000000 /* yes. 1M insns */ #define MAX_TAIL_CALL_CNT 33 /* Maximum number of loops for bpf_loop and bpf_iter_num. * It's enum to expose it (and thus make it discoverable) through BTF. */ enum { BPF_MAX_LOOPS = 8 * 1024 * 1024, BPF_MAX_TIMED_LOOPS = 0xffff, }; #define BPF_F_ACCESS_MASK (BPF_F_RDONLY | \ BPF_F_RDONLY_PROG | \ BPF_F_WRONLY | \ BPF_F_WRONLY_PROG) #define BPF_MAP_CAN_READ BIT(0) #define BPF_MAP_CAN_WRITE BIT(1) /* Maximum number of user-producer ring buffer samples that can be drained in * a call to bpf_user_ringbuf_drain(). */ #define BPF_MAX_USER_RINGBUF_SAMPLES (128 * 1024) static inline u32 bpf_map_flags_to_cap(struct bpf_map *map) { u32 access_flags = map->map_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); /* Combination of BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG is * not possible. */ if (access_flags & BPF_F_RDONLY_PROG) return BPF_MAP_CAN_READ; else if (access_flags & BPF_F_WRONLY_PROG) return BPF_MAP_CAN_WRITE; else return BPF_MAP_CAN_READ | BPF_MAP_CAN_WRITE; } static inline bool bpf_map_flags_access_ok(u32 access_flags) { return (access_flags & (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG)) != (BPF_F_RDONLY_PROG | BPF_F_WRONLY_PROG); } static inline struct bpf_map_owner *bpf_map_owner_alloc(struct bpf_map *map) { return kzalloc(sizeof(*map->owner), GFP_ATOMIC); } static inline void bpf_map_owner_free(struct bpf_map *map) { kfree(map->owner); } struct bpf_event_entry { struct perf_event *event; struct file *perf_file; struct file *map_file; struct rcu_head rcu; }; static inline bool map_type_contains_progs(struct bpf_map *map) { return map->map_type == BPF_MAP_TYPE_PROG_ARRAY || map->map_type == BPF_MAP_TYPE_DEVMAP || map->map_type == BPF_MAP_TYPE_CPUMAP; } bool bpf_prog_map_compatible(struct bpf_map *map, const struct bpf_prog *fp); int bpf_prog_calc_tag(struct bpf_prog *fp); const struct bpf_func_proto *bpf_get_trace_printk_proto(void); const struct bpf_func_proto *bpf_get_trace_vprintk_proto(void); const struct bpf_func_proto *bpf_get_perf_event_read_value_proto(void); typedef unsigned long (*bpf_ctx_copy_t)(void *dst, const void *src, unsigned long off, unsigned long len); typedef u32 (*bpf_convert_ctx_access_t)(enum bpf_access_type type, const struct bpf_insn *src, struct bpf_insn *dst, struct bpf_prog *prog, u32 *target_size); u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy); /* an array of programs to be executed under rcu_lock. * * Typical usage: * ret = bpf_prog_run_array(rcu_dereference(&bpf_prog_array), ctx, bpf_prog_run); * * the structure returned by bpf_prog_array_alloc() should be populated * with program pointers and the last pointer must be NULL. * The user has to keep refcnt on the program and make sure the program * is removed from the array before bpf_prog_put(). * The 'struct bpf_prog_array *' should only be replaced with xchg() * since other cpus are walking the array of pointers in parallel. */ struct bpf_prog_array_item { struct bpf_prog *prog; union { struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; u64 bpf_cookie; }; }; struct bpf_prog_array { struct rcu_head rcu; struct bpf_prog_array_item items[]; }; struct bpf_empty_prog_array { struct bpf_prog_array hdr; struct bpf_prog *null_prog; }; /* to avoid allocating empty bpf_prog_array for cgroups that * don't have bpf program attached use one global 'bpf_empty_prog_array' * It will not be modified the caller of bpf_prog_array_alloc() * (since caller requested prog_cnt == 0) * that pointer should be 'freed' by bpf_prog_array_free() */ extern struct bpf_empty_prog_array bpf_empty_prog_array; struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); void bpf_prog_array_free(struct bpf_prog_array *progs); /* Use when traversal over the bpf_prog_array uses tasks_trace rcu */ void bpf_prog_array_free_sleepable(struct bpf_prog_array *progs); int bpf_prog_array_length(struct bpf_prog_array *progs); bool bpf_prog_array_is_empty(struct bpf_prog_array *array); int bpf_prog_array_copy_to_user(struct bpf_prog_array *progs, __u32 __user *prog_ids, u32 cnt); void bpf_prog_array_delete_safe(struct bpf_prog_array *progs, struct bpf_prog *old_prog); int bpf_prog_array_delete_safe_at(struct bpf_prog_array *array, int index); int bpf_prog_array_update_at(struct bpf_prog_array *array, int index, struct bpf_prog *prog); int bpf_prog_array_copy_info(struct bpf_prog_array *array, u32 *prog_ids, u32 request_cnt, u32 *prog_cnt); int bpf_prog_array_copy(struct bpf_prog_array *old_array, struct bpf_prog *exclude_prog, struct bpf_prog *include_prog, u64 bpf_cookie, struct bpf_prog_array **new_array); struct bpf_run_ctx {}; struct bpf_cg_run_ctx { struct bpf_run_ctx run_ctx; const struct bpf_prog_array_item *prog_item; int retval; }; struct bpf_trace_run_ctx { struct bpf_run_ctx run_ctx; u64 bpf_cookie; bool is_uprobe; }; struct bpf_tramp_run_ctx { struct bpf_run_ctx run_ctx; u64 bpf_cookie; struct bpf_run_ctx *saved_run_ctx; }; static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx) { struct bpf_run_ctx *old_ctx = NULL; #ifdef CONFIG_BPF_SYSCALL old_ctx = current->bpf_ctx; current->bpf_ctx = new_ctx; #endif return old_ctx; } static inline void bpf_reset_run_ctx(struct bpf_run_ctx *old_ctx) { #ifdef CONFIG_BPF_SYSCALL current->bpf_ctx = old_ctx; #endif } /* BPF program asks to bypass CAP_NET_BIND_SERVICE in bind. */ #define BPF_RET_BIND_NO_CAP_NET_BIND_SERVICE (1 << 0) /* BPF program asks to set CN on the packet. */ #define BPF_RET_SET_CN (1 << 0) typedef u32 (*bpf_prog_run_fn)(const struct bpf_prog *prog, const void *ctx); static __always_inline u32 bpf_prog_run_array(const struct bpf_prog_array *array, const void *ctx, bpf_prog_run_fn run_prog) { const struct bpf_prog_array_item *item; const struct bpf_prog *prog; struct bpf_run_ctx *old_run_ctx; struct bpf_trace_run_ctx run_ctx; u32 ret = 1; RCU_LOCKDEP_WARN(!rcu_read_lock_held(), "no rcu lock held"); if (unlikely(!array)) return ret; run_ctx.is_uprobe = false; migrate_disable(); old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); item = &array->items[0]; while ((prog = READ_ONCE(item->prog))) { run_ctx.bpf_cookie = item->bpf_cookie; ret &= run_prog(prog, ctx); item++; } bpf_reset_run_ctx(old_run_ctx); migrate_enable(); return ret; } /* Notes on RCU design for bpf_prog_arrays containing sleepable programs: * * We use the tasks_trace rcu flavor read section to protect the bpf_prog_array * overall. As a result, we must use the bpf_prog_array_free_sleepable * in order to use the tasks_trace rcu grace period. * * When a non-sleepable program is inside the array, we take the rcu read * section and disable preemption for that program alone, so it can access * rcu-protected dynamically sized maps. */ static __always_inline u32 bpf_prog_run_array_uprobe(const struct bpf_prog_array *array, const void *ctx, bpf_prog_run_fn run_prog) { const struct bpf_prog_array_item *item; const struct bpf_prog *prog; struct bpf_run_ctx *old_run_ctx; struct bpf_trace_run_ctx run_ctx; u32 ret = 1; might_fault(); RCU_LOCKDEP_WARN(!rcu_read_lock_trace_held(), "no rcu lock held"); if (unlikely(!array)) return ret; migrate_disable(); run_ctx.is_uprobe = true; old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); item = &array->items[0]; while ((prog = READ_ONCE(item->prog))) { if (!prog->sleepable) rcu_read_lock(); run_ctx.bpf_cookie = item->bpf_cookie; ret &= run_prog(prog, ctx); item++; if (!prog->sleepable) rcu_read_unlock(); } bpf_reset_run_ctx(old_run_ctx); migrate_enable(); return ret; } bool bpf_jit_bypass_spec_v1(void); bool bpf_jit_bypass_spec_v4(void); #ifdef CONFIG_BPF_SYSCALL DECLARE_PER_CPU(int, bpf_prog_active); extern struct mutex bpf_stats_enabled_mutex; /* * Block execution of BPF programs attached to instrumentation (perf, * kprobes, tracepoints) to prevent deadlocks on map operations as any of * these events can happen inside a region which holds a map bucket lock * and can deadlock on it. */ static inline void bpf_disable_instrumentation(void) { migrate_disable(); this_cpu_inc(bpf_prog_active); } static inline void bpf_enable_instrumentation(void) { this_cpu_dec(bpf_prog_active); migrate_enable(); } extern const struct super_operations bpf_super_ops; extern const struct file_operations bpf_map_fops; extern const struct file_operations bpf_prog_fops; extern const struct file_operations bpf_iter_fops; extern const struct file_operations bpf_token_fops; #define BPF_PROG_TYPE(_id, _name, prog_ctx_type, kern_ctx_type) \ extern const struct bpf_prog_ops _name ## _prog_ops; \ extern const struct bpf_verifier_ops _name ## _verifier_ops; #define BPF_MAP_TYPE(_id, _ops) \ extern const struct bpf_map_ops _ops; #define BPF_LINK_TYPE(_id, _name) #include <linux/bpf_types.h> #undef BPF_PROG_TYPE #undef BPF_MAP_TYPE #undef BPF_LINK_TYPE extern const struct bpf_prog_ops bpf_offload_prog_ops; extern const struct bpf_verifier_ops tc_cls_act_analyzer_ops; extern const struct bpf_verifier_ops xdp_analyzer_ops; struct bpf_prog *bpf_prog_get(u32 ufd); struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, bool attach_drv); void bpf_prog_add(struct bpf_prog *prog, int i); void bpf_prog_sub(struct bpf_prog *prog, int i); void bpf_prog_inc(struct bpf_prog *prog); struct bpf_prog * __must_check bpf_prog_inc_not_zero(struct bpf_prog *prog); void bpf_prog_put(struct bpf_prog *prog); void bpf_prog_free_id(struct bpf_prog *prog); void bpf_map_free_id(struct bpf_map *map); struct btf_field *btf_record_find(const struct btf_record *rec, u32 offset, u32 field_mask); void btf_record_free(struct btf_record *rec); void bpf_map_free_record(struct bpf_map *map); struct btf_record *btf_record_dup(const struct btf_record *rec); bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *rec_b); void bpf_obj_free_timer(const struct btf_record *rec, void *obj); void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj); void bpf_obj_free_task_work(const struct btf_record *rec, void *obj); void bpf_obj_free_fields(const struct btf_record *rec, void *obj); void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu); struct bpf_map *bpf_map_get(u32 ufd); struct bpf_map *bpf_map_get_with_uref(u32 ufd); /* * The __bpf_map_get() and __btf_get_by_fd() functions parse a file * descriptor and return a corresponding map or btf object. * Their names are double underscored to emphasize the fact that they * do not increase refcnt. To also increase refcnt use corresponding * bpf_map_get() and btf_get_by_fd() functions. */ static inline struct bpf_map *__bpf_map_get(struct fd f) { if (fd_empty(f)) return ERR_PTR(-EBADF); if (unlikely(fd_file(f)->f_op != &bpf_map_fops)) return ERR_PTR(-EINVAL); return fd_file(f)->private_data; } static inline struct btf *__btf_get_by_fd(struct fd f) { if (fd_empty(f)) return ERR_PTR(-EBADF); if (unlikely(fd_file(f)->f_op != &btf_fops)) return ERR_PTR(-EINVAL); return fd_file(f)->private_data; } void bpf_map_inc(struct bpf_map *map); void bpf_map_inc_with_uref(struct bpf_map *map); struct bpf_map *__bpf_map_inc_not_zero(struct bpf_map *map, bool uref); struct bpf_map * __must_check bpf_map_inc_not_zero(struct bpf_map *map); void bpf_map_put_with_uref(struct bpf_map *map); void bpf_map_put(struct bpf_map *map); void *bpf_map_area_alloc(u64 size, int numa_node); void *bpf_map_area_mmapable_alloc(u64 size, int numa_node); void bpf_map_area_free(void *base); bool bpf_map_write_active(const struct bpf_map *map); void bpf_map_init_from_attr(struct bpf_map *map, union bpf_attr *attr); int generic_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, union bpf_attr __user *uattr); int generic_map_update_batch(struct bpf_map *map, struct file *map_file, const union bpf_attr *attr, union bpf_attr __user *uattr); int generic_map_delete_batch(struct bpf_map *map, const union bpf_attr *attr, union bpf_attr __user *uattr); struct bpf_map *bpf_map_get_curr_or_next(u32 *id); struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); int bpf_map_alloc_pages(const struct bpf_map *map, int nid, unsigned long nr_pages, struct page **page_array); #ifdef CONFIG_MEMCG void *bpf_map_kmalloc_node(const struct bpf_map *map, size_t size, gfp_t flags, int node); void *bpf_map_kmalloc_nolock(const struct bpf_map *map, size_t size, gfp_t flags, int node); void *bpf_map_kzalloc(const struct bpf_map *map, size_t size, gfp_t flags); void *bpf_map_kvcalloc(struct bpf_map *map, size_t n, size_t size, gfp_t flags); void __percpu *bpf_map_alloc_percpu(const struct bpf_map *map, size_t size, size_t align, gfp_t flags); #else /* * These specialized allocators have to be macros for their allocations to be * accounted separately (to have separate alloc_tag). */ #define bpf_map_kmalloc_node(_map, _size, _flags, _node) \ kmalloc_node(_size, _flags, _node) #define bpf_map_kmalloc_nolock(_map, _size, _flags, _node) \ kmalloc_nolock(_size, _flags, _node) #define bpf_map_kzalloc(_map, _size, _flags) \ kzalloc(_size, _flags) #define bpf_map_kvcalloc(_map, _n, _size, _flags) \ kvcalloc(_n, _size, _flags) #define bpf_map_alloc_percpu(_map, _size, _align, _flags) \ __alloc_percpu_gfp(_size, _align, _flags) #endif static inline int bpf_map_init_elem_count(struct bpf_map *map) { size_t size = sizeof(*map->elem_count), align = size; gfp_t flags = GFP_USER | __GFP_NOWARN; map->elem_count = bpf_map_alloc_percpu(map, size, align, flags); if (!map->elem_count) return -ENOMEM; return 0; } static inline void bpf_map_free_elem_count(struct bpf_map *map) { free_percpu(map->elem_count); } static inline void bpf_map_inc_elem_count(struct bpf_map *map) { this_cpu_inc(*map->elem_count); } static inline void bpf_map_dec_elem_count(struct bpf_map *map) { this_cpu_dec(*map->elem_count); } extern int sysctl_unprivileged_bpf_disabled; bool bpf_token_capable(const struct bpf_token *token, int cap); static inline bool bpf_allow_ptr_leaks(const struct bpf_token *token) { return bpf_token_capable(token, CAP_PERFMON); } static inline bool bpf_allow_uninit_stack(const struct bpf_token *token) { return bpf_token_capable(token, CAP_PERFMON); } static inline bool bpf_bypass_spec_v1(const struct bpf_token *token) { return bpf_jit_bypass_spec_v1() || cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON); } static inline bool bpf_bypass_spec_v4(const struct bpf_token *token) { return bpf_jit_bypass_spec_v4() || cpu_mitigations_off() || bpf_token_capable(token, CAP_PERFMON); } int bpf_map_new_fd(struct bpf_map *map, int flags); int bpf_prog_new_fd(struct bpf_prog *prog); void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, const struct bpf_link_ops *ops, struct bpf_prog *prog, enum bpf_attach_type attach_type); void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type, const struct bpf_link_ops *ops, struct bpf_prog *prog, enum bpf_attach_type attach_type, bool sleepable); int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer); int bpf_link_settle(struct bpf_link_primer *primer); void bpf_link_cleanup(struct bpf_link_primer *primer); void bpf_link_inc(struct bpf_link *link); struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link); void bpf_link_put(struct bpf_link *link); int bpf_link_new_fd(struct bpf_link *link); struct bpf_link *bpf_link_get_from_fd(u32 ufd); struct bpf_link *bpf_link_get_curr_or_next(u32 *id); void bpf_token_inc(struct bpf_token *token); void bpf_token_put(struct bpf_token *token); int bpf_token_create(union bpf_attr *attr); struct bpf_token *bpf_token_get_from_fd(u32 ufd); int bpf_token_get_info_by_fd(struct bpf_token *token, const union bpf_attr *attr, union bpf_attr __user *uattr); bool bpf_token_allow_cmd(const struct bpf_token *token, enum bpf_cmd cmd); bool bpf_token_allow_map_type(const struct bpf_token *token, enum bpf_map_type type); bool bpf_token_allow_prog_type(const struct bpf_token *token, enum bpf_prog_type prog_type, enum bpf_attach_type attach_type); int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname); int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags); struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir, umode_t mode); #define BPF_ITER_FUNC_PREFIX "bpf_iter_" #define DEFINE_BPF_ITER_FUNC(target, args...) \ extern int bpf_iter_ ## target(args); \ int __init bpf_iter_ ## target(args) { return 0; } /* * The task type of iterators. * * For BPF task iterators, they can be parameterized with various * parameters to visit only some of tasks. * * BPF_TASK_ITER_ALL (default) * Iterate over resources of every task. * * BPF_TASK_ITER_TID * Iterate over resources of a task/tid. * * BPF_TASK_ITER_TGID * Iterate over resources of every task of a process / task group. */ enum bpf_iter_task_type { BPF_TASK_ITER_ALL = 0, BPF_TASK_ITER_TID, BPF_TASK_ITER_TGID, }; struct bpf_iter_aux_info { /* for map_elem iter */ struct bpf_map *map; /* for cgroup iter */ struct { struct cgroup *start; /* starting cgroup */ enum bpf_cgroup_iter_order order; } cgroup; struct { enum bpf_iter_task_type type; u32 pid; } task; }; typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux, struct seq_file *seq); typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux, struct bpf_link_info *info); typedef const struct bpf_func_proto * (*bpf_iter_get_func_proto_t)(enum bpf_func_id func_id, const struct bpf_prog *prog); enum bpf_iter_feature { BPF_ITER_RESCHED = BIT(0), }; #define BPF_ITER_CTX_ARG_MAX 2 struct bpf_iter_reg { const char *target; bpf_iter_attach_target_t attach_target; bpf_iter_detach_target_t detach_target; bpf_iter_show_fdinfo_t show_fdinfo; bpf_iter_fill_link_info_t fill_link_info; bpf_iter_get_func_proto_t get_func_proto; u32 ctx_arg_info_size; u32 feature; struct bpf_ctx_arg_aux ctx_arg_info[BPF_ITER_CTX_ARG_MAX]; const struct bpf_iter_seq_info *seq_info; }; struct bpf_iter_meta { __bpf_md_ptr(struct seq_file *, seq); u64 session_id; u64 seq_num; }; struct bpf_iter__bpf_map_elem { __bpf_md_ptr(struct bpf_iter_meta *, meta); __bpf_md_ptr(struct bpf_map *, map); __bpf_md_ptr(void *, key); __bpf_md_ptr(void *, value); }; int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info); void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info); int bpf_iter_prog_supported(struct bpf_prog *prog); const struct bpf_func_proto * bpf_iter_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); int bpf_iter_link_attach(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_prog *prog); int bpf_iter_new_fd(struct bpf_link *link); bool bpf_link_is_iter(struct bpf_link *link); struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop); int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx); void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, struct seq_file *seq); int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, struct bpf_link_info *info); int map_set_for_each_callback_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, struct bpf_func_state *callee); int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value); int bpf_percpu_array_copy(struct bpf_map *map, void *key, void *value); int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, u64 flags); int bpf_percpu_array_update(struct bpf_map *map, void *key, void *value, u64 flags); int bpf_stackmap_extract(struct bpf_map *map, void *key, void *value, bool delete); int bpf_fd_array_map_update_elem(struct bpf_map *map, struct file *map_file, void *key, void *value, u64 map_flags); int bpf_fd_array_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, void *key, void *value, u64 map_flags); int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value); int bpf_get_file_flag(int flags); int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size, size_t actual_size); /* verify correctness of eBPF program */ int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size); #ifndef CONFIG_BPF_JIT_ALWAYS_ON void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); #endif struct btf *bpf_get_btf_vmlinux(void); /* Map specifics */ struct xdp_frame; struct sk_buff; struct bpf_dtab_netdev; struct bpf_cpu_map_entry; void __dev_flush(struct list_head *flush_list); int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, struct net_device *dev_rx); int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, struct net_device *dev_rx); int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, struct bpf_map *map, bool exclude_ingress); int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, const struct bpf_prog *xdp_prog); int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, const struct bpf_prog *xdp_prog, struct bpf_map *map, bool exclude_ingress); void __cpu_map_flush(struct list_head *flush_list); int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, struct net_device *dev_rx); int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, struct sk_buff *skb); /* Return map's numa specified by userspace */ static inline int bpf_map_attr_numa_node(const union bpf_attr *attr) { return (attr->map_flags & BPF_F_NUMA_NODE) ? attr->numa_node : NUMA_NO_NODE; } struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type); int array_map_alloc_check(union bpf_attr *attr); int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); int bpf_prog_test_run_tracing(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); int bpf_prog_test_run_raw_tp(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); int bpf_prog_test_run_nf(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); bool btf_ctx_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info); static inline bool bpf_tracing_ctx_access(int off, int size, enum bpf_access_type type) { if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS) return false; if (type != BPF_READ) return false; if (off % size != 0) return false; return true; } static inline bool bpf_tracing_btf_ctx_access(int off, int size, enum bpf_access_type type, const struct bpf_prog *prog, struct bpf_insn_access_aux *info) { if (!bpf_tracing_ctx_access(off, size, type)) return false; return btf_ctx_access(off, size, type, prog, info); } int btf_struct_access(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, int off, int size, enum bpf_access_type atype, u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name); bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, bool strict); int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf *btf, const struct btf_type *func_proto, const char *func_name, struct btf_func_model *m); struct bpf_reg_state; int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog); int btf_check_type_match(struct bpf_verifier_log *log, const struct bpf_prog *prog, struct btf *btf, const struct btf_type *t); const char *btf_find_decl_tag_value(const struct btf *btf, const struct btf_type *pt, int comp_idx, const char *tag_key); int btf_find_next_decl_tag(const struct btf *btf, const struct btf_type *pt, int comp_idx, const char *tag_key, int last_id); struct bpf_prog *bpf_prog_by_id(u32 id); struct bpf_link *bpf_link_by_id(u32 id); const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); void bpf_task_storage_free(struct task_struct *task); void bpf_cgrp_storage_free(struct cgroup *cgroup); bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, u16 btf_fd_idx, u8 **func_addr); struct bpf_core_ctx { struct bpf_verifier_log *log; const struct btf *btf; }; bool btf_nested_type_is_trusted(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, const char *field_name, u32 btf_id, const char *suffix); bool btf_type_ids_nocast_alias(struct bpf_verifier_log *log, const struct btf *reg_btf, u32 reg_id, const struct btf *arg_btf, u32 arg_id); int bpf_core_apply(struct bpf_core_ctx *ctx, const struct bpf_core_relo *relo, int relo_idx, void *insn); static inline bool unprivileged_ebpf_enabled(void) { return !sysctl_unprivileged_bpf_disabled; } /* Not all bpf prog type has the bpf_ctx. * For the bpf prog type that has initialized the bpf_ctx, * this function can be used to decide if a kernel function * is called by a bpf program. */ static inline bool has_current_bpf_ctx(void) { return !!current->bpf_ctx; } void notrace bpf_prog_inc_misses_counter(struct bpf_prog *prog); void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data, enum bpf_dynptr_type type, u32 offset, u32 size); void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr); void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr); void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip); #else /* !CONFIG_BPF_SYSCALL */ static inline struct bpf_prog *bpf_prog_get(u32 ufd) { return ERR_PTR(-EOPNOTSUPP); } static inline struct bpf_prog *bpf_prog_get_type_dev(u32 ufd, enum bpf_prog_type type, bool attach_drv) { return ERR_PTR(-EOPNOTSUPP); } static inline void bpf_prog_add(struct bpf_prog *prog, int i) { } static inline void bpf_prog_sub(struct bpf_prog *prog, int i) { } static inline void bpf_prog_put(struct bpf_prog *prog) { } static inline void bpf_prog_inc(struct bpf_prog *prog) { } static inline struct bpf_prog *__must_check bpf_prog_inc_not_zero(struct bpf_prog *prog) { return ERR_PTR(-EOPNOTSUPP); } static inline void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, const struct bpf_link_ops *ops, struct bpf_prog *prog, enum bpf_attach_type attach_type) { } static inline void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type, const struct bpf_link_ops *ops, struct bpf_prog *prog, enum bpf_attach_type attach_type, bool sleepable) { } static inline int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) { return -EOPNOTSUPP; } static inline int bpf_link_settle(struct bpf_link_primer *primer) { return -EOPNOTSUPP; } static inline void bpf_link_cleanup(struct bpf_link_primer *primer) { } static inline void bpf_link_inc(struct bpf_link *link) { } static inline struct bpf_link *bpf_link_inc_not_zero(struct bpf_link *link) { return NULL; } static inline void bpf_link_put(struct bpf_link *link) { } static inline int bpf_obj_get_user(const char __user *pathname, int flags) { return -EOPNOTSUPP; } static inline bool bpf_token_capable(const struct bpf_token *token, int cap) { return capable(cap) || (cap != CAP_SYS_ADMIN && capable(CAP_SYS_ADMIN)); } static inline void bpf_token_inc(struct bpf_token *token) { } static inline void bpf_token_put(struct bpf_token *token) { } static inline struct bpf_token *bpf_token_get_from_fd(u32 ufd) { return ERR_PTR(-EOPNOTSUPP); } static inline int bpf_token_get_info_by_fd(struct bpf_token *token, const union bpf_attr *attr, union bpf_attr __user *uattr) { return -EOPNOTSUPP; } static inline void __dev_flush(struct list_head *flush_list) { } struct xdp_frame; struct bpf_dtab_netdev; struct bpf_cpu_map_entry; static inline int dev_xdp_enqueue(struct net_device *dev, struct xdp_frame *xdpf, struct net_device *dev_rx) { return 0; } static inline int dev_map_enqueue(struct bpf_dtab_netdev *dst, struct xdp_frame *xdpf, struct net_device *dev_rx) { return 0; } static inline int dev_map_enqueue_multi(struct xdp_frame *xdpf, struct net_device *dev_rx, struct bpf_map *map, bool exclude_ingress) { return 0; } struct sk_buff; static inline int dev_map_generic_redirect(struct bpf_dtab_netdev *dst, struct sk_buff *skb, const struct bpf_prog *xdp_prog) { return 0; } static inline int dev_map_redirect_multi(struct net_device *dev, struct sk_buff *skb, const struct bpf_prog *xdp_prog, struct bpf_map *map, bool exclude_ingress) { return 0; } static inline void __cpu_map_flush(struct list_head *flush_list) { } static inline int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, struct net_device *dev_rx) { return 0; } static inline int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, struct sk_buff *skb) { return -EOPNOTSUPP; } static inline struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type) { return ERR_PTR(-EOPNOTSUPP); } static inline int bpf_prog_test_run_xdp(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr) { return -ENOTSUPP; } static inline int bpf_prog_test_run_skb(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr) { return -ENOTSUPP; } static inline int bpf_prog_test_run_tracing(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr) { return -ENOTSUPP; } static inline int bpf_prog_test_run_flow_dissector(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr) { return -ENOTSUPP; } static inline int bpf_prog_test_run_sk_lookup(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr) { return -ENOTSUPP; } static inline void bpf_map_put(struct bpf_map *map) { } static inline struct bpf_prog *bpf_prog_by_id(u32 id) { return ERR_PTR(-ENOTSUPP); } static inline int btf_struct_access(struct bpf_verifier_log *log, const struct bpf_reg_state *reg, int off, int size, enum bpf_access_type atype, u32 *next_btf_id, enum bpf_type_flag *flag, const char **field_name) { return -EACCES; } static inline const struct bpf_func_proto * bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) { return NULL; } static inline void bpf_task_storage_free(struct task_struct *task) { } static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) { return false; } static inline const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) { return NULL; } static inline int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, u16 btf_fd_idx, u8 **func_addr) { return -ENOTSUPP; } static inline bool unprivileged_ebpf_enabled(void) { return false; } static inline bool has_current_bpf_ctx(void) { return false; } static inline void bpf_prog_inc_misses_counter(struct bpf_prog *prog) { } static inline void bpf_cgrp_storage_free(struct cgroup *cgroup) { } static inline void bpf_dynptr_init(struct bpf_dynptr_kern *ptr, void *data, enum bpf_dynptr_type type, u32 offset, u32 size) { } static inline void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr) { } static inline void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr) { } static inline void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip) { } #endif /* CONFIG_BPF_SYSCALL */ static __always_inline int bpf_probe_read_kernel_common(void *dst, u32 size, const void *unsafe_ptr) { int ret = -EFAULT; if (IS_ENABLED(CONFIG_BPF_EVENTS)) ret = copy_from_kernel_nofault(dst, unsafe_ptr, size); if (unlikely(ret < 0)) memset(dst, 0, size); return ret; } void __bpf_free_used_btfs(struct btf_mod_pair *used_btfs, u32 len); static inline struct bpf_prog *bpf_prog_get_type(u32 ufd, enum bpf_prog_type type) { return bpf_prog_get_type_dev(ufd, type, false); } void __bpf_free_used_maps(struct bpf_prog_aux *aux, struct bpf_map **used_maps, u32 len); bool bpf_prog_get_ok(struct bpf_prog *, enum bpf_prog_type *, bool); int bpf_prog_offload_compile(struct bpf_prog *prog); void bpf_prog_dev_bound_destroy(struct bpf_prog *prog); int bpf_prog_offload_info_fill(struct bpf_prog_info *info, struct bpf_prog *prog); int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map); int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value); int bpf_map_offload_update_elem(struct bpf_map *map, void *key, void *value, u64 flags); int bpf_map_offload_delete_elem(struct bpf_map *map, void *key); int bpf_map_offload_get_next_key(struct bpf_map *map, void *key, void *next_key); bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map); struct bpf_offload_dev * bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv); void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev); void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev); int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev, struct net_device *netdev); void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev, struct net_device *netdev); bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev); void unpriv_ebpf_notify(int new_state); #if defined(CONFIG_NET) && defined(CONFIG_BPF_SYSCALL) int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log, struct bpf_prog_aux *prog_aux); void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id); int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr); int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog); void bpf_dev_bound_netdev_unregister(struct net_device *dev); static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) { return aux->dev_bound; } static inline bool bpf_prog_is_offloaded(const struct bpf_prog_aux *aux) { return aux->offload_requested; } bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs); static inline bool bpf_map_is_offloaded(struct bpf_map *map) { return unlikely(map->ops == &bpf_map_offload_ops); } struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr); void bpf_map_offload_map_free(struct bpf_map *map); u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map); int bpf_prog_test_run_syscall(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog); int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags); int sock_map_bpf_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr); int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog); void sock_map_unhash(struct sock *sk); void sock_map_destroy(struct sock *sk); void sock_map_close(struct sock *sk, long timeout); #else static inline int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log, struct bpf_prog_aux *prog_aux) { return -EOPNOTSUPP; } static inline void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id) { return NULL; } static inline int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr) { return -EOPNOTSUPP; } static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog) { return -EOPNOTSUPP; } static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev) { } static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) { return false; } static inline bool bpf_prog_is_offloaded(struct bpf_prog_aux *aux) { return false; } static inline bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs) { return false; } static inline bool bpf_map_is_offloaded(struct bpf_map *map) { return false; } static inline struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr) { return ERR_PTR(-EOPNOTSUPP); } static inline void bpf_map_offload_map_free(struct bpf_map *map) { } static inline u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map) { return 0; } static inline int bpf_prog_test_run_syscall(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr) { return -ENOTSUPP; } #ifdef CONFIG_BPF_SYSCALL static inline int sock_map_get_from_fd(const union bpf_attr *attr, struct bpf_prog *prog) { return -EINVAL; } static inline int sock_map_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype) { return -EOPNOTSUPP; } static inline int sock_map_update_elem_sys(struct bpf_map *map, void *key, void *value, u64 flags) { return -EOPNOTSUPP; } static inline int sock_map_bpf_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr) { return -EINVAL; } static inline int sock_map_link_create(const union bpf_attr *attr, struct bpf_prog *prog) { return -EOPNOTSUPP; } #endif /* CONFIG_BPF_SYSCALL */ #endif /* CONFIG_NET && CONFIG_BPF_SYSCALL */ static __always_inline void bpf_prog_inc_misses_counters(const struct bpf_prog_array *array) { const struct bpf_prog_array_item *item; struct bpf_prog *prog; if (unlikely(!array)) return; item = &array->items[0]; while ((prog = READ_ONCE(item->prog))) { bpf_prog_inc_misses_counter(prog); item++; } } #if defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) void bpf_sk_reuseport_detach(struct sock *sk); int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, void *value); int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, void *value, u64 map_flags); #else static inline void bpf_sk_reuseport_detach(struct sock *sk) { } #ifdef CONFIG_BPF_SYSCALL static inline int bpf_fd_reuseport_array_lookup_elem(struct bpf_map *map, void *key, void *value) { return -EOPNOTSUPP; } static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, void *key, void *value, u64 map_flags) { return -EOPNOTSUPP; } #endif /* CONFIG_BPF_SYSCALL */ #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ #if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p, struct bpf_dynptr *sig_p, struct bpf_key *trusted_keyring); #else static inline struct bpf_key *bpf_lookup_user_key(u32 serial, u64 flags) { return NULL; } static inline struct bpf_key *bpf_lookup_system_key(u64 id) { return NULL; } static inline void bpf_key_put(struct bpf_key *bkey) { } static inline int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p, struct bpf_dynptr *sig_p, struct bpf_key *trusted_keyring) { return -EOPNOTSUPP; } #endif /* defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) */ /* verifier prototypes for helper functions called from eBPF programs */ extern const struct bpf_func_proto bpf_map_lookup_elem_proto; extern const struct bpf_func_proto bpf_map_update_elem_proto; extern const struct bpf_func_proto bpf_map_delete_elem_proto; extern const struct bpf_func_proto bpf_map_push_elem_proto; extern const struct bpf_func_proto bpf_map_pop_elem_proto; extern const struct bpf_func_proto bpf_map_peek_elem_proto; extern const struct bpf_func_proto bpf_map_lookup_percpu_elem_proto; extern const struct bpf_func_proto bpf_get_prandom_u32_proto; extern const struct bpf_func_proto bpf_get_smp_processor_id_proto; extern const struct bpf_func_proto bpf_get_numa_node_id_proto; extern const struct bpf_func_proto bpf_tail_call_proto; extern const struct bpf_func_proto bpf_ktime_get_ns_proto; extern const struct bpf_func_proto bpf_ktime_get_boot_ns_proto; extern const struct bpf_func_proto bpf_ktime_get_tai_ns_proto; extern const struct bpf_func_proto bpf_get_current_pid_tgid_proto; extern const struct bpf_func_proto bpf_get_current_uid_gid_proto; extern const struct bpf_func_proto bpf_get_current_comm_proto; extern const struct bpf_func_proto bpf_get_stackid_proto; extern const struct bpf_func_proto bpf_get_stack_proto; extern const struct bpf_func_proto bpf_get_stack_sleepable_proto; extern const struct bpf_func_proto bpf_get_task_stack_proto; extern const struct bpf_func_proto bpf_get_task_stack_sleepable_proto; extern const struct bpf_func_proto bpf_get_stackid_proto_pe; extern const struct bpf_func_proto bpf_get_stack_proto_pe; extern const struct bpf_func_proto bpf_sock_map_update_proto; extern const struct bpf_func_proto bpf_sock_hash_update_proto; extern const struct bpf_func_proto bpf_get_current_cgroup_id_proto; extern const struct bpf_func_proto bpf_get_current_ancestor_cgroup_id_proto; extern const struct bpf_func_proto bpf_get_cgroup_classid_curr_proto; extern const struct bpf_func_proto bpf_current_task_under_cgroup_proto; extern const struct bpf_func_proto bpf_msg_redirect_hash_proto; extern const struct bpf_func_proto bpf_msg_redirect_map_proto; extern const struct bpf_func_proto bpf_sk_redirect_hash_proto; extern const struct bpf_func_proto bpf_sk_redirect_map_proto; extern const struct bpf_func_proto bpf_spin_lock_proto; extern const struct bpf_func_proto bpf_spin_unlock_proto; extern const struct bpf_func_proto bpf_get_local_storage_proto; extern const struct bpf_func_proto bpf_strtol_proto; extern const struct bpf_func_proto bpf_strtoul_proto; extern const struct bpf_func_proto bpf_tcp_sock_proto; extern const struct bpf_func_proto bpf_jiffies64_proto; extern const struct bpf_func_proto bpf_get_ns_current_pid_tgid_proto; extern const struct bpf_func_proto bpf_event_output_data_proto; extern const struct bpf_func_proto bpf_ringbuf_output_proto; extern const struct bpf_func_proto bpf_ringbuf_reserve_proto; extern const struct bpf_func_proto bpf_ringbuf_submit_proto; extern const struct bpf_func_proto bpf_ringbuf_discard_proto; extern const struct bpf_func_proto bpf_ringbuf_query_proto; extern const struct bpf_func_proto bpf_ringbuf_reserve_dynptr_proto; extern const struct bpf_func_proto bpf_ringbuf_submit_dynptr_proto; extern const struct bpf_func_proto bpf_ringbuf_discard_dynptr_proto; extern const struct bpf_func_proto bpf_skc_to_tcp6_sock_proto; extern const struct bpf_func_proto bpf_skc_to_tcp_sock_proto; extern const struct bpf_func_proto bpf_skc_to_tcp_timewait_sock_proto; extern const struct bpf_func_proto bpf_skc_to_tcp_request_sock_proto; extern const struct bpf_func_proto bpf_skc_to_udp6_sock_proto; extern const struct bpf_func_proto bpf_skc_to_unix_sock_proto; extern const struct bpf_func_proto bpf_skc_to_mptcp_sock_proto; extern const struct bpf_func_proto bpf_copy_from_user_proto; extern const struct bpf_func_proto bpf_snprintf_btf_proto; extern const struct bpf_func_proto bpf_snprintf_proto; extern const struct bpf_func_proto bpf_per_cpu_ptr_proto; extern const struct bpf_func_proto bpf_this_cpu_ptr_proto; extern const struct bpf_func_proto bpf_ktime_get_coarse_ns_proto; extern const struct bpf_func_proto bpf_sock_from_file_proto; extern const struct bpf_func_proto bpf_get_socket_ptr_cookie_proto; extern const struct bpf_func_proto bpf_task_storage_get_recur_proto; extern const struct bpf_func_proto bpf_task_storage_get_proto; extern const struct bpf_func_proto bpf_task_storage_delete_recur_proto; extern const struct bpf_func_proto bpf_task_storage_delete_proto; extern const struct bpf_func_proto bpf_for_each_map_elem_proto; extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto; extern const struct bpf_func_proto bpf_sk_setsockopt_proto; extern const struct bpf_func_proto bpf_sk_getsockopt_proto; extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto; extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto; extern const struct bpf_func_proto bpf_find_vma_proto; extern const struct bpf_func_proto bpf_loop_proto; extern const struct bpf_func_proto bpf_copy_from_user_task_proto; extern const struct bpf_func_proto bpf_set_retval_proto; extern const struct bpf_func_proto bpf_get_retval_proto; extern const struct bpf_func_proto bpf_user_ringbuf_drain_proto; extern const struct bpf_func_proto bpf_cgrp_storage_get_proto; extern const struct bpf_func_proto bpf_cgrp_storage_delete_proto; const struct bpf_func_proto *tracing_prog_func_proto( enum bpf_func_id func_id, const struct bpf_prog *prog); /* Shared helpers among cBPF and eBPF. */ void bpf_user_rnd_init_once(void); u64 bpf_user_rnd_u32(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); u64 bpf_get_raw_cpu_id(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); #if defined(CONFIG_NET) bool bpf_sock_common_is_valid_access(int off, int size, enum bpf_access_type type, struct bpf_insn_access_aux *info); bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, struct bpf_insn_access_aux *info); u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, const struct bpf_insn *si, struct bpf_insn *insn_buf, struct bpf_prog *prog, u32 *target_size); int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags, struct bpf_dynptr *ptr); #else static inline bool bpf_sock_common_is_valid_access(int off, int size, enum bpf_access_type type, struct bpf_insn_access_aux *info) { return false; } static inline bool bpf_sock_is_valid_access(int off, int size, enum bpf_access_type type, struct bpf_insn_access_aux *info) { return false; } static inline u32 bpf_sock_convert_ctx_access(enum bpf_access_type type, const struct bpf_insn *si, struct bpf_insn *insn_buf, struct bpf_prog *prog, u32 *target_size) { return 0; } static inline int bpf_dynptr_from_skb_rdonly(struct __sk_buff *skb, u64 flags, struct bpf_dynptr *ptr) { return -EOPNOTSUPP; } #endif #ifdef CONFIG_INET struct sk_reuseport_kern { struct sk_buff *skb; struct sock *sk; struct sock *selected_sk; struct sock *migrating_sk; void *data_end; u32 hash; u32 reuseport_id; bool bind_inany; }; bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, struct bpf_insn_access_aux *info); u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, const struct bpf_insn *si, struct bpf_insn *insn_buf, struct bpf_prog *prog, u32 *target_size); bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, struct bpf_insn_access_aux *info); u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, const struct bpf_insn *si, struct bpf_insn *insn_buf, struct bpf_prog *prog, u32 *target_size); #else static inline bool bpf_tcp_sock_is_valid_access(int off, int size, enum bpf_access_type type, struct bpf_insn_access_aux *info) { return false; } static inline u32 bpf_tcp_sock_convert_ctx_access(enum bpf_access_type type, const struct bpf_insn *si, struct bpf_insn *insn_buf, struct bpf_prog *prog, u32 *target_size) { return 0; } static inline bool bpf_xdp_sock_is_valid_access(int off, int size, enum bpf_access_type type, struct bpf_insn_access_aux *info) { return false; } static inline u32 bpf_xdp_sock_convert_ctx_access(enum bpf_access_type type, const struct bpf_insn *si, struct bpf_insn *insn_buf, struct bpf_prog *prog, u32 *target_size) { return 0; } #endif /* CONFIG_INET */ enum bpf_text_poke_type { BPF_MOD_CALL, BPF_MOD_JUMP, }; int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, void *addr1, void *addr2); void bpf_arch_poke_desc_update(struct bpf_jit_poke_descriptor *poke, struct bpf_prog *new, struct bpf_prog *old); void *bpf_arch_text_copy(void *dst, void *src, size_t len); int bpf_arch_text_invalidate(void *dst, size_t len); struct btf_id_set; bool btf_id_set_contains(const struct btf_id_set *set, u32 id); #define MAX_BPRINTF_VARARGS 12 #define MAX_BPRINTF_BUF 1024 /* Per-cpu temp buffers used by printf-like helpers to store the bprintf binary * arguments representation. */ #define MAX_BPRINTF_BIN_ARGS 512 struct bpf_bprintf_buffers { char bin_args[MAX_BPRINTF_BIN_ARGS]; char buf[MAX_BPRINTF_BUF]; }; struct bpf_bprintf_data { u32 *bin_args; char *buf; bool get_bin_args; bool get_buf; }; int bpf_bprintf_prepare(const char *fmt, u32 fmt_size, const u64 *raw_args, u32 num_args, struct bpf_bprintf_data *data); void bpf_bprintf_cleanup(struct bpf_bprintf_data *data); int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs); void bpf_put_buffers(void); void bpf_prog_stream_init(struct bpf_prog *prog); void bpf_prog_stream_free(struct bpf_prog *prog); int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len); void bpf_stream_stage_init(struct bpf_stream_stage *ss); void bpf_stream_stage_free(struct bpf_stream_stage *ss); __printf(2, 3) int bpf_stream_stage_printk(struct bpf_stream_stage *ss, const char *fmt, ...); int bpf_stream_stage_commit(struct bpf_stream_stage *ss, struct bpf_prog *prog, enum bpf_stream_id stream_id); int bpf_stream_stage_dump_stack(struct bpf_stream_stage *ss); #define bpf_stream_printk(ss, ...) bpf_stream_stage_printk(&ss, __VA_ARGS__) #define bpf_stream_dump_stack(ss) bpf_stream_stage_dump_stack(&ss) #define bpf_stream_stage(ss, prog, stream_id, expr) \ ({ \ bpf_stream_stage_init(&ss); \ (expr); \ bpf_stream_stage_commit(&ss, prog, stream_id); \ bpf_stream_stage_free(&ss); \ }) #ifdef CONFIG_BPF_LSM void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype); void bpf_cgroup_atype_put(int cgroup_atype); #else static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {} static inline void bpf_cgroup_atype_put(int cgroup_atype) {} #endif /* CONFIG_BPF_LSM */ struct key; #ifdef CONFIG_KEYS struct bpf_key { struct key *key; bool has_ref; }; #endif /* CONFIG_KEYS */ static inline bool type_is_alloc(u32 type) { return type & MEM_ALLOC; } static inline gfp_t bpf_memcg_flags(gfp_t flags) { if (memcg_bpf_enabled()) return flags | __GFP_ACCOUNT; return flags; } static inline bool bpf_is_subprog(const struct bpf_prog *prog) { return prog->aux->func_idx != 0; } int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump); struct bpf_prog *bpf_prog_find_from_stack(void); #endif /* _LINUX_BPF_H */ |
| 1461 1460 1337 207 2 1599 2 5 4 1 4 4 4 2 2 4 2 1 1 2 1 1 1 212 208 13 5 5 75 75 75 1521 1519 1398 255 1454 1336 204 3 1 1 1 1 1 919 313 301 13 919 919 75 1533 1529 921 314 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 | // SPDX-License-Identifier: GPL-2.0 /* * Management Component Transport Protocol (MCTP) - device implementation. * * Copyright (c) 2021 Code Construct * Copyright (c) 2021 Google */ #include <linux/if_arp.h> #include <linux/if_link.h> #include <linux/mctp.h> #include <linux/netdevice.h> #include <linux/rcupdate.h> #include <linux/rtnetlink.h> #include <net/addrconf.h> #include <net/netlink.h> #include <net/mctp.h> #include <net/mctpdevice.h> #include <net/sock.h> struct mctp_dump_cb { unsigned long ifindex; size_t a_idx; }; /* unlocked: caller must hold rcu_read_lock. * Returned mctp_dev has its refcount incremented, or NULL if unset. */ struct mctp_dev *__mctp_dev_get(const struct net_device *dev) { struct mctp_dev *mdev = rcu_dereference(dev->mctp_ptr); /* RCU guarantees that any mdev is still live. * Zero refcount implies a pending free, return NULL. */ if (mdev) if (!refcount_inc_not_zero(&mdev->refs)) return NULL; return mdev; } /* Returned mctp_dev does not have refcount incremented. The returned pointer * remains live while rtnl_lock is held, as that prevents mctp_unregister() */ struct mctp_dev *mctp_dev_get_rtnl(const struct net_device *dev) { return rtnl_dereference(dev->mctp_ptr); } static int mctp_addrinfo_size(void) { return NLMSG_ALIGN(sizeof(struct ifaddrmsg)) + nla_total_size(1) // IFA_LOCAL + nla_total_size(1) // IFA_ADDRESS ; } /* flag should be NLM_F_MULTI for dump calls */ static int mctp_fill_addrinfo(struct sk_buff *skb, struct mctp_dev *mdev, mctp_eid_t eid, int msg_type, u32 portid, u32 seq, int flag) { struct ifaddrmsg *hdr; struct nlmsghdr *nlh; nlh = nlmsg_put(skb, portid, seq, msg_type, sizeof(*hdr), flag); if (!nlh) return -EMSGSIZE; hdr = nlmsg_data(nlh); hdr->ifa_family = AF_MCTP; hdr->ifa_prefixlen = 0; hdr->ifa_flags = 0; hdr->ifa_scope = 0; hdr->ifa_index = mdev->dev->ifindex; if (nla_put_u8(skb, IFA_LOCAL, eid)) goto cancel; if (nla_put_u8(skb, IFA_ADDRESS, eid)) goto cancel; nlmsg_end(skb, nlh); return 0; cancel: nlmsg_cancel(skb, nlh); return -EMSGSIZE; } static int mctp_dump_dev_addrinfo(struct mctp_dev *mdev, struct sk_buff *skb, struct netlink_callback *cb) { struct mctp_dump_cb *mcb = (void *)cb->ctx; u32 portid, seq; int rc = 0; portid = NETLINK_CB(cb->skb).portid; seq = cb->nlh->nlmsg_seq; for (; mcb->a_idx < mdev->num_addrs; mcb->a_idx++) { rc = mctp_fill_addrinfo(skb, mdev, mdev->addrs[mcb->a_idx], RTM_NEWADDR, portid, seq, NLM_F_MULTI); if (rc < 0) break; } return rc; } static int mctp_dump_addrinfo(struct sk_buff *skb, struct netlink_callback *cb) { struct mctp_dump_cb *mcb = (void *)cb->ctx; struct net *net = sock_net(skb->sk); struct net_device *dev; struct ifaddrmsg *hdr; struct mctp_dev *mdev; int ifindex = 0, rc; /* Filter by ifindex if a header is provided */ hdr = nlmsg_payload(cb->nlh, sizeof(*hdr)); if (hdr) { ifindex = hdr->ifa_index; } else { if (cb->strict_check) { NL_SET_ERR_MSG(cb->extack, "mctp: Invalid header for addr dump request"); return -EINVAL; } } rcu_read_lock(); for_each_netdev_dump(net, dev, mcb->ifindex) { if (ifindex && ifindex != dev->ifindex) continue; mdev = __mctp_dev_get(dev); if (!mdev) continue; rc = mctp_dump_dev_addrinfo(mdev, skb, cb); mctp_dev_put(mdev); if (rc < 0) break; mcb->a_idx = 0; } rcu_read_unlock(); return skb->len; } static void mctp_addr_notify(struct mctp_dev *mdev, mctp_eid_t eid, int msg_type, struct sk_buff *req_skb, struct nlmsghdr *req_nlh) { u32 portid = NETLINK_CB(req_skb).portid; struct net *net = dev_net(mdev->dev); struct sk_buff *skb; int rc = -ENOBUFS; skb = nlmsg_new(mctp_addrinfo_size(), GFP_KERNEL); if (!skb) goto out; rc = mctp_fill_addrinfo(skb, mdev, eid, msg_type, portid, req_nlh->nlmsg_seq, 0); if (rc < 0) { WARN_ON_ONCE(rc == -EMSGSIZE); goto out; } rtnl_notify(skb, net, portid, RTNLGRP_MCTP_IFADDR, req_nlh, GFP_KERNEL); return; out: kfree_skb(skb); rtnl_set_sk_err(net, RTNLGRP_MCTP_IFADDR, rc); } static const struct nla_policy ifa_mctp_policy[IFA_MAX + 1] = { [IFA_ADDRESS] = { .type = NLA_U8 }, [IFA_LOCAL] = { .type = NLA_U8 }, }; static int mctp_rtm_newaddr(struct sk_buff *skb, struct nlmsghdr *nlh, struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct nlattr *tb[IFA_MAX + 1]; struct net_device *dev; struct mctp_addr *addr; struct mctp_dev *mdev; struct ifaddrmsg *ifm; unsigned long flags; u8 *tmp_addrs; int rc; rc = nlmsg_parse(nlh, sizeof(*ifm), tb, IFA_MAX, ifa_mctp_policy, extack); if (rc < 0) return rc; ifm = nlmsg_data(nlh); if (tb[IFA_LOCAL]) addr = nla_data(tb[IFA_LOCAL]); else if (tb[IFA_ADDRESS]) addr = nla_data(tb[IFA_ADDRESS]); else return -EINVAL; /* find device */ dev = __dev_get_by_index(net, ifm->ifa_index); if (!dev) return -ENODEV; mdev = mctp_dev_get_rtnl(dev); if (!mdev) return -ENODEV; if (!mctp_address_unicast(addr->s_addr)) return -EINVAL; /* Prevent duplicates. Under RTNL so don't need to lock for reading */ if (memchr(mdev->addrs, addr->s_addr, mdev->num_addrs)) return -EEXIST; tmp_addrs = kmalloc(mdev->num_addrs + 1, GFP_KERNEL); if (!tmp_addrs) return -ENOMEM; memcpy(tmp_addrs, mdev->addrs, mdev->num_addrs); tmp_addrs[mdev->num_addrs] = addr->s_addr; /* Lock to write */ spin_lock_irqsave(&mdev->addrs_lock, flags); mdev->num_addrs++; swap(mdev->addrs, tmp_addrs); spin_unlock_irqrestore(&mdev->addrs_lock, flags); kfree(tmp_addrs); mctp_addr_notify(mdev, addr->s_addr, RTM_NEWADDR, skb, nlh); mctp_route_add_local(mdev, addr->s_addr); return 0; } static int mctp_rtm_deladdr(struct sk_buff *skb, struct nlmsghdr *nlh, struct netlink_ext_ack *extack) { struct net *net = sock_net(skb->sk); struct nlattr *tb[IFA_MAX + 1]; struct net_device *dev; struct mctp_addr *addr; struct mctp_dev *mdev; struct ifaddrmsg *ifm; unsigned long flags; u8 *pos; int rc; rc = nlmsg_parse(nlh, sizeof(*ifm), tb, IFA_MAX, ifa_mctp_policy, extack); if (rc < 0) return rc; ifm = nlmsg_data(nlh); if (tb[IFA_LOCAL]) addr = nla_data(tb[IFA_LOCAL]); else if (tb[IFA_ADDRESS]) addr = nla_data(tb[IFA_ADDRESS]); else return -EINVAL; /* find device */ dev = __dev_get_by_index(net, ifm->ifa_index); if (!dev) return -ENODEV; mdev = mctp_dev_get_rtnl(dev); if (!mdev) return -ENODEV; pos = memchr(mdev->addrs, addr->s_addr, mdev->num_addrs); if (!pos) return -ENOENT; rc = mctp_route_remove_local(mdev, addr->s_addr); // we can ignore -ENOENT in the case a route was already removed if (rc < 0 && rc != -ENOENT) return rc; spin_lock_irqsave(&mdev->addrs_lock, flags); memmove(pos, pos + 1, mdev->num_addrs - 1 - (pos - mdev->addrs)); mdev->num_addrs--; spin_unlock_irqrestore(&mdev->addrs_lock, flags); mctp_addr_notify(mdev, addr->s_addr, RTM_DELADDR, skb, nlh); return 0; } void mctp_dev_hold(struct mctp_dev *mdev) { refcount_inc(&mdev->refs); } void mctp_dev_put(struct mctp_dev *mdev) { if (mdev && refcount_dec_and_test(&mdev->refs)) { kfree(mdev->addrs); dev_put(mdev->dev); kfree_rcu(mdev, rcu); } } void mctp_dev_release_key(struct mctp_dev *dev, struct mctp_sk_key *key) __must_hold(&key->lock) { if (!dev) return; if (dev->ops && dev->ops->release_flow) dev->ops->release_flow(dev, key); key->dev = NULL; mctp_dev_put(dev); } void mctp_dev_set_key(struct mctp_dev *dev, struct mctp_sk_key *key) __must_hold(&key->lock) { mctp_dev_hold(dev); key->dev = dev; } static struct mctp_dev *mctp_add_dev(struct net_device *dev) { struct mctp_dev *mdev; ASSERT_RTNL(); mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); if (!mdev) return ERR_PTR(-ENOMEM); spin_lock_init(&mdev->addrs_lock); mdev->net = mctp_default_net(dev_net(dev)); /* associate to net_device */ refcount_set(&mdev->refs, 1); rcu_assign_pointer(dev->mctp_ptr, mdev); dev_hold(dev); mdev->dev = dev; return mdev; } static int mctp_fill_link_af(struct sk_buff *skb, const struct net_device *dev, u32 ext_filter_mask) { struct mctp_dev *mdev; mdev = mctp_dev_get_rtnl(dev); if (!mdev) return -ENODATA; if (nla_put_u32(skb, IFLA_MCTP_NET, mdev->net)) return -EMSGSIZE; if (nla_put_u8(skb, IFLA_MCTP_PHYS_BINDING, mdev->binding)) return -EMSGSIZE; return 0; } static size_t mctp_get_link_af_size(const struct net_device *dev, u32 ext_filter_mask) { struct mctp_dev *mdev; unsigned int ret; /* caller holds RCU */ mdev = __mctp_dev_get(dev); if (!mdev) return 0; ret = nla_total_size(4); /* IFLA_MCTP_NET */ ret += nla_total_size(1); /* IFLA_MCTP_PHYS_BINDING */ mctp_dev_put(mdev); return ret; } static const struct nla_policy ifla_af_mctp_policy[IFLA_MCTP_MAX + 1] = { [IFLA_MCTP_NET] = { .type = NLA_U32 }, }; static int mctp_set_link_af(struct net_device *dev, const struct nlattr *attr, struct netlink_ext_ack *extack) { struct nlattr *tb[IFLA_MCTP_MAX + 1]; struct mctp_dev *mdev; int rc; rc = nla_parse_nested(tb, IFLA_MCTP_MAX, attr, ifla_af_mctp_policy, NULL); if (rc) return rc; mdev = mctp_dev_get_rtnl(dev); if (!mdev) return 0; if (tb[IFLA_MCTP_NET]) WRITE_ONCE(mdev->net, nla_get_u32(tb[IFLA_MCTP_NET])); return 0; } /* Matches netdev types that should have MCTP handling */ static bool mctp_known(struct net_device *dev) { /* only register specific types (inc. NONE for TUN devices) */ return dev->type == ARPHRD_MCTP || dev->type == ARPHRD_LOOPBACK || dev->type == ARPHRD_NONE; } static void mctp_unregister(struct net_device *dev) { struct mctp_dev *mdev; mdev = mctp_dev_get_rtnl(dev); if (!mdev) return; RCU_INIT_POINTER(mdev->dev->mctp_ptr, NULL); mctp_route_remove_dev(mdev); mctp_neigh_remove_dev(mdev); mctp_dev_put(mdev); } static int mctp_register(struct net_device *dev) { struct mctp_dev *mdev; /* Already registered? */ if (rtnl_dereference(dev->mctp_ptr)) return 0; /* only register specific types */ if (!mctp_known(dev)) return 0; mdev = mctp_add_dev(dev); if (IS_ERR(mdev)) return PTR_ERR(mdev); return 0; } static int mctp_dev_notify(struct notifier_block *this, unsigned long event, void *ptr) { struct net_device *dev = netdev_notifier_info_to_dev(ptr); int rc; switch (event) { case NETDEV_REGISTER: rc = mctp_register(dev); if (rc) return notifier_from_errno(rc); break; case NETDEV_UNREGISTER: mctp_unregister(dev); break; } return NOTIFY_OK; } static int mctp_register_netdevice(struct net_device *dev, const struct mctp_netdev_ops *ops, enum mctp_phys_binding binding) { struct mctp_dev *mdev; mdev = mctp_add_dev(dev); if (IS_ERR(mdev)) return PTR_ERR(mdev); mdev->ops = ops; mdev->binding = binding; return register_netdevice(dev); } int mctp_register_netdev(struct net_device *dev, const struct mctp_netdev_ops *ops, enum mctp_phys_binding binding) { int rc; rtnl_lock(); rc = mctp_register_netdevice(dev, ops, binding); rtnl_unlock(); return rc; } EXPORT_SYMBOL_GPL(mctp_register_netdev); void mctp_unregister_netdev(struct net_device *dev) { unregister_netdev(dev); } EXPORT_SYMBOL_GPL(mctp_unregister_netdev); static struct rtnl_af_ops mctp_af_ops = { .family = AF_MCTP, .fill_link_af = mctp_fill_link_af, .get_link_af_size = mctp_get_link_af_size, .set_link_af = mctp_set_link_af, }; static struct notifier_block mctp_dev_nb = { .notifier_call = mctp_dev_notify, .priority = ADDRCONF_NOTIFY_PRIORITY, }; static const struct rtnl_msg_handler mctp_device_rtnl_msg_handlers[] = { {.owner = THIS_MODULE, .protocol = PF_MCTP, .msgtype = RTM_NEWADDR, .doit = mctp_rtm_newaddr}, {.owner = THIS_MODULE, .protocol = PF_MCTP, .msgtype = RTM_DELADDR, .doit = mctp_rtm_deladdr}, {.owner = THIS_MODULE, .protocol = PF_MCTP, .msgtype = RTM_GETADDR, .dumpit = mctp_dump_addrinfo}, }; int __init mctp_device_init(void) { int err; register_netdevice_notifier(&mctp_dev_nb); err = rtnl_af_register(&mctp_af_ops); if (err) goto err_notifier; err = rtnl_register_many(mctp_device_rtnl_msg_handlers); if (err) goto err_af; return 0; err_af: rtnl_af_unregister(&mctp_af_ops); err_notifier: unregister_netdevice_notifier(&mctp_dev_nb); return err; } void __exit mctp_device_exit(void) { rtnl_unregister_many(mctp_device_rtnl_msg_handlers); rtnl_af_unregister(&mctp_af_ops); unregister_netdevice_notifier(&mctp_dev_nb); } |
| 5 5 5 2 3 5 5 5 5 2 3 2 3 5 5 5 5 5 2 3 5 5 5 5 5 5 5 5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 | // SPDX-License-Identifier: GPL-2.0+ /* * usbduxsigma.c * Copyright (C) 2011-2015 Bernd Porr, mail@berndporr.me.uk */ /* * Driver: usbduxsigma * Description: University of Stirling USB DAQ & INCITE Technology Limited * Devices: [ITL] USB-DUX-SIGMA (usbduxsigma) * Author: Bernd Porr <mail@berndporr.me.uk> * Updated: 20 July 2015 * Status: stable */ /* * I must give credit here to Chris Baugher who * wrote the driver for AT-MIO-16d. I used some parts of this * driver. I also must give credits to David Brownell * who supported me with the USB development. * * Note: the raw data from the A/D converter is 24 bit big endian * anything else is little endian to/from the dux board * * * Revision history: * 0.1: initial version * 0.2: all basic functions implemented, digital I/O only for one port * 0.3: proper vendor ID and driver name * 0.4: fixed D/A voltage range * 0.5: various bug fixes, health check at startup * 0.6: corrected wrong input range * 0.7: rewrite code that urb->interval is always 1 */ #include <linux/kernel.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/input.h> #include <linux/fcntl.h> #include <linux/compiler.h> #include <linux/unaligned.h> #include <linux/comedi/comedi_usb.h> /* timeout for the USB-transfer in ms*/ #define BULK_TIMEOUT 1000 /* constants for "firmware" upload and download */ #define FIRMWARE "usbduxsigma_firmware.bin" #define FIRMWARE_MAX_LEN 0x4000 #define USBDUXSUB_FIRMWARE 0xa0 #define VENDOR_DIR_IN 0xc0 #define VENDOR_DIR_OUT 0x40 /* internal addresses of the 8051 processor */ #define USBDUXSUB_CPUCS 0xE600 /* 300Hz max frequ under PWM */ #define MIN_PWM_PERIOD ((long)(1E9 / 300)) /* Default PWM frequency */ #define PWM_DEFAULT_PERIOD ((long)(1E9 / 100)) /* Number of channels (16 AD and offset)*/ #define NUMCHANNELS 16 /* Size of one A/D value */ #define SIZEADIN ((sizeof(u32))) /* * Size of the async input-buffer IN BYTES, the DIO state is transmitted * as the first byte. */ #define SIZEINBUF (((NUMCHANNELS + 1) * SIZEADIN)) /* 16 bytes. */ #define SIZEINSNBUF 16 /* Number of DA channels */ #define NUMOUTCHANNELS 8 /* size of one value for the D/A converter: channel and value */ #define SIZEDAOUT ((sizeof(u8) + sizeof(uint16_t))) /* * Size of the output-buffer in bytes * Actually only the first 4 triplets are used but for the * high speed mode we need to pad it to 8 (microframes). */ #define SIZEOUTBUF ((8 * SIZEDAOUT)) /* * Size of the buffer for the dux commands: just now max size is determined * by the analogue out + command byte + panic bytes... */ #define SIZEOFDUXBUFFER ((8 * SIZEDAOUT + 2)) /* Number of in-URBs which receive the data: min=2 */ #define NUMOFINBUFFERSFULL 5 /* Number of out-URBs which send the data: min=2 */ #define NUMOFOUTBUFFERSFULL 5 /* Number of in-URBs which receive the data: min=5 */ /* must have more buffers due to buggy USB ctr */ #define NUMOFINBUFFERSHIGH 10 /* Number of out-URBs which send the data: min=5 */ /* must have more buffers due to buggy USB ctr */ #define NUMOFOUTBUFFERSHIGH 10 /* number of retries to get the right dux command */ #define RETRIES 10 /* bulk transfer commands to usbduxsigma */ #define USBBUXSIGMA_AD_CMD 9 #define USBDUXSIGMA_DA_CMD 1 #define USBDUXSIGMA_DIO_CFG_CMD 2 #define USBDUXSIGMA_DIO_BITS_CMD 3 #define USBDUXSIGMA_SINGLE_AD_CMD 4 #define USBDUXSIGMA_PWM_ON_CMD 7 #define USBDUXSIGMA_PWM_OFF_CMD 8 static const struct comedi_lrange usbduxsigma_ai_range = { 1, { BIP_RANGE(2.5 * 0x800000 / 0x780000 / 2.0) } }; struct usbduxsigma_private { /* actual number of in-buffers */ int n_ai_urbs; /* actual number of out-buffers */ int n_ao_urbs; /* ISO-transfer handling: buffers */ struct urb **ai_urbs; struct urb **ao_urbs; /* pwm-transfer handling */ struct urb *pwm_urb; /* PWM period */ unsigned int pwm_period; /* PWM internal delay for the GPIF in the FX2 */ u8 pwm_delay; /* size of the PWM buffer which holds the bit pattern */ int pwm_buf_sz; /* input buffer for the ISO-transfer */ __be32 *in_buf; /* input buffer for single insn */ u8 *insn_buf; unsigned high_speed:1; unsigned ai_cmd_running:1; unsigned ao_cmd_running:1; unsigned pwm_cmd_running:1; /* time between samples in units of the timer */ unsigned int ai_timer; unsigned int ao_timer; /* counter between acquisitions */ unsigned int ai_counter; unsigned int ao_counter; /* interval in frames/uframes */ unsigned int ai_interval; /* commands */ u8 *dux_commands; struct mutex mut; }; static void usbduxsigma_unlink_urbs(struct urb **urbs, int num_urbs) { int i; for (i = 0; i < num_urbs; i++) usb_kill_urb(urbs[i]); } static void usbduxsigma_ai_stop(struct comedi_device *dev, int do_unlink) { struct usbduxsigma_private *devpriv = dev->private; if (do_unlink && devpriv->ai_urbs) usbduxsigma_unlink_urbs(devpriv->ai_urbs, devpriv->n_ai_urbs); devpriv->ai_cmd_running = 0; } static int usbduxsigma_ai_cancel(struct comedi_device *dev, struct comedi_subdevice *s) { struct usbduxsigma_private *devpriv = dev->private; mutex_lock(&devpriv->mut); /* unlink only if it is really running */ usbduxsigma_ai_stop(dev, devpriv->ai_cmd_running); mutex_unlock(&devpriv->mut); return 0; } static void usbduxsigma_ai_handle_urb(struct comedi_device *dev, struct comedi_subdevice *s, struct urb *urb) { struct usbduxsigma_private *devpriv = dev->private; struct comedi_async *async = s->async; struct comedi_cmd *cmd = &async->cmd; u32 val; int ret; int i; if ((urb->actual_length > 0) && (urb->status != -EXDEV)) { devpriv->ai_counter--; if (devpriv->ai_counter == 0) { devpriv->ai_counter = devpriv->ai_timer; /* * Get the data from the USB bus and hand it over * to comedi. Note, first byte is the DIO state. */ for (i = 0; i < cmd->chanlist_len; i++) { val = be32_to_cpu(devpriv->in_buf[i + 1]); val &= 0x00ffffff; /* strip status byte */ val = comedi_offset_munge(s, val); if (!comedi_buf_write_samples(s, &val, 1)) return; } if (cmd->stop_src == TRIG_COUNT && async->scans_done >= cmd->stop_arg) async->events |= COMEDI_CB_EOA; } } /* if command is still running, resubmit urb */ if (!(async->events & COMEDI_CB_CANCEL_MASK)) { urb->dev = comedi_to_usb_dev(dev); ret = usb_submit_urb(urb, GFP_ATOMIC); if (ret < 0) { dev_err(dev->class_dev, "urb resubmit failed (%d)\n", ret); if (ret == -EL2NSYNC) dev_err(dev->class_dev, "buggy USB host controller or bug in IRQ handler\n"); async->events |= COMEDI_CB_ERROR; } } } static void usbduxsigma_ai_urb_complete(struct urb *urb) { struct comedi_device *dev = urb->context; struct usbduxsigma_private *devpriv = dev->private; struct comedi_subdevice *s = dev->read_subdev; struct comedi_async *async = s->async; /* exit if not running a command, do not resubmit urb */ if (!devpriv->ai_cmd_running) return; switch (urb->status) { case 0: /* copy the result in the transfer buffer */ memcpy(devpriv->in_buf, urb->transfer_buffer, SIZEINBUF); usbduxsigma_ai_handle_urb(dev, s, urb); break; case -EILSEQ: /* * error in the ISOchronous data * we don't copy the data into the transfer buffer * and recycle the last data byte */ dev_dbg(dev->class_dev, "CRC error in ISO IN stream\n"); usbduxsigma_ai_handle_urb(dev, s, urb); break; case -ECONNRESET: case -ENOENT: case -ESHUTDOWN: case -ECONNABORTED: /* happens after an unlink command */ async->events |= COMEDI_CB_ERROR; break; default: /* a real error */ dev_err(dev->class_dev, "non-zero urb status (%d)\n", urb->status); async->events |= COMEDI_CB_ERROR; break; } /* * comedi_handle_events() cannot be used in this driver. The (*cancel) * operation would unlink the urb. */ if (async->events & COMEDI_CB_CANCEL_MASK) usbduxsigma_ai_stop(dev, 0); comedi_event(dev, s); } static void usbduxsigma_ao_stop(struct comedi_device *dev, int do_unlink) { struct usbduxsigma_private *devpriv = dev->private; if (do_unlink && devpriv->ao_urbs) usbduxsigma_unlink_urbs(devpriv->ao_urbs, devpriv->n_ao_urbs); devpriv->ao_cmd_running = 0; } static int usbduxsigma_ao_cancel(struct comedi_device *dev, struct comedi_subdevice *s) { struct usbduxsigma_private *devpriv = dev->private; mutex_lock(&devpriv->mut); /* unlink only if it is really running */ usbduxsigma_ao_stop(dev, devpriv->ao_cmd_running); mutex_unlock(&devpriv->mut); return 0; } static void usbduxsigma_ao_handle_urb(struct comedi_device *dev, struct comedi_subdevice *s, struct urb *urb) { struct usbduxsigma_private *devpriv = dev->private; struct comedi_async *async = s->async; struct comedi_cmd *cmd = &async->cmd; u8 *datap; int ret; int i; devpriv->ao_counter--; if (devpriv->ao_counter == 0) { devpriv->ao_counter = devpriv->ao_timer; if (cmd->stop_src == TRIG_COUNT && async->scans_done >= cmd->stop_arg) { async->events |= COMEDI_CB_EOA; return; } /* transmit data to the USB bus */ datap = urb->transfer_buffer; *datap++ = cmd->chanlist_len; for (i = 0; i < cmd->chanlist_len; i++) { unsigned int chan = CR_CHAN(cmd->chanlist[i]); unsigned short val; if (!comedi_buf_read_samples(s, &val, 1)) { dev_err(dev->class_dev, "buffer underflow\n"); async->events |= COMEDI_CB_OVERFLOW; return; } *datap++ = val; *datap++ = chan; s->readback[chan] = val; } } /* if command is still running, resubmit urb */ if (!(async->events & COMEDI_CB_CANCEL_MASK)) { urb->transfer_buffer_length = SIZEOUTBUF; urb->dev = comedi_to_usb_dev(dev); urb->status = 0; urb->interval = 1; /* (u)frames */ urb->number_of_packets = 1; urb->iso_frame_desc[0].offset = 0; urb->iso_frame_desc[0].length = SIZEOUTBUF; urb->iso_frame_desc[0].status = 0; ret = usb_submit_urb(urb, GFP_ATOMIC); if (ret < 0) { dev_err(dev->class_dev, "urb resubmit failed (%d)\n", ret); if (ret == -EL2NSYNC) dev_err(dev->class_dev, "buggy USB host controller or bug in IRQ handler\n"); async->events |= COMEDI_CB_ERROR; } } } static void usbduxsigma_ao_urb_complete(struct urb *urb) { struct comedi_device *dev = urb->context; struct usbduxsigma_private *devpriv = dev->private; struct comedi_subdevice *s = dev->write_subdev; struct comedi_async *async = s->async; /* exit if not running a command, do not resubmit urb */ if (!devpriv->ao_cmd_running) return; switch (urb->status) { case 0: usbduxsigma_ao_handle_urb(dev, s, urb); break; case -ECONNRESET: case -ENOENT: case -ESHUTDOWN: case -ECONNABORTED: /* happens after an unlink command */ async->events |= COMEDI_CB_ERROR; break; default: /* a real error */ dev_err(dev->class_dev, "non-zero urb status (%d)\n", urb->status); async->events |= COMEDI_CB_ERROR; break; } /* * comedi_handle_events() cannot be used in this driver. The (*cancel) * operation would unlink the urb. */ if (async->events & COMEDI_CB_CANCEL_MASK) usbduxsigma_ao_stop(dev, 0); comedi_event(dev, s); } static int usbduxsigma_submit_urbs(struct comedi_device *dev, struct urb **urbs, int num_urbs, int input_urb) { struct usb_device *usb = comedi_to_usb_dev(dev); struct urb *urb; int ret; int i; /* Submit all URBs and start the transfer on the bus */ for (i = 0; i < num_urbs; i++) { urb = urbs[i]; /* in case of a resubmission after an unlink... */ if (input_urb) urb->interval = 1; urb->context = dev; urb->dev = usb; urb->status = 0; urb->transfer_flags = URB_ISO_ASAP; ret = usb_submit_urb(urb, GFP_ATOMIC); if (ret) return ret; } return 0; } static int usbduxsigma_chans_to_interval(int num_chan) { if (num_chan <= 2) return 2; /* 4kHz */ if (num_chan <= 8) return 4; /* 2kHz */ return 8; /* 1kHz */ } static int usbduxsigma_ai_cmdtest(struct comedi_device *dev, struct comedi_subdevice *s, struct comedi_cmd *cmd) { struct usbduxsigma_private *devpriv = dev->private; int high_speed = devpriv->high_speed; int interval = usbduxsigma_chans_to_interval(cmd->chanlist_len); unsigned int tmp; int err = 0; /* Step 1 : check if triggers are trivially valid */ err |= comedi_check_trigger_src(&cmd->start_src, TRIG_NOW | TRIG_INT); err |= comedi_check_trigger_src(&cmd->scan_begin_src, TRIG_TIMER); err |= comedi_check_trigger_src(&cmd->convert_src, TRIG_NOW); err |= comedi_check_trigger_src(&cmd->scan_end_src, TRIG_COUNT); err |= comedi_check_trigger_src(&cmd->stop_src, TRIG_COUNT | TRIG_NONE); if (err) return 1; /* Step 2a : make sure trigger sources are unique */ err |= comedi_check_trigger_is_unique(cmd->start_src); err |= comedi_check_trigger_is_unique(cmd->stop_src); /* Step 2b : and mutually compatible */ if (err) return 2; /* Step 3: check if arguments are trivially valid */ err |= comedi_check_trigger_arg_is(&cmd->start_arg, 0); if (high_speed) { /* * In high speed mode microframes are possible. * However, during one microframe we can roughly * sample two channels. Thus, the more channels * are in the channel list the more time we need. */ err |= comedi_check_trigger_arg_min(&cmd->scan_begin_arg, (125000 * interval)); } else { /* full speed */ /* 1kHz scans every USB frame */ err |= comedi_check_trigger_arg_min(&cmd->scan_begin_arg, 1000000); } err |= comedi_check_trigger_arg_is(&cmd->scan_end_arg, cmd->chanlist_len); if (cmd->stop_src == TRIG_COUNT) err |= comedi_check_trigger_arg_min(&cmd->stop_arg, 1); else /* TRIG_NONE */ err |= comedi_check_trigger_arg_is(&cmd->stop_arg, 0); if (err) return 3; /* Step 4: fix up any arguments */ tmp = rounddown(cmd->scan_begin_arg, high_speed ? 125000 : 1000000); err |= comedi_check_trigger_arg_is(&cmd->scan_begin_arg, tmp); if (err) return 4; return 0; } /* * creates the ADC command for the MAX1271 * range is the range value from comedi */ static void create_adc_command(unsigned int chan, u8 *muxsg0, u8 *muxsg1) { if (chan < 8) (*muxsg0) = (*muxsg0) | (1 << chan); else if (chan < 16) (*muxsg1) = (*muxsg1) | (1 << (chan - 8)); } static int usbbuxsigma_send_cmd(struct comedi_device *dev, int cmd_type) { struct usb_device *usb = comedi_to_usb_dev(dev); struct usbduxsigma_private *devpriv = dev->private; int nsent; devpriv->dux_commands[0] = cmd_type; return usb_bulk_msg(usb, usb_sndbulkpipe(usb, 1), devpriv->dux_commands, SIZEOFDUXBUFFER, &nsent, BULK_TIMEOUT); } static int usbduxsigma_receive_cmd(struct comedi_device *dev, int command) { struct usb_device *usb = comedi_to_usb_dev(dev); struct usbduxsigma_private *devpriv = dev->private; int nrec; int ret; int i; for (i = 0; i < RETRIES; i++) { ret = usb_bulk_msg(usb, usb_rcvbulkpipe(usb, 8), devpriv->insn_buf, SIZEINSNBUF, &nrec, BULK_TIMEOUT); if (ret < 0) return ret; if (devpriv->insn_buf[0] == command) return 0; } /* * This is only reached if the data has been requested a * couple of times and the command was not received. */ return -EFAULT; } static int usbduxsigma_ai_inttrig(struct comedi_device *dev, struct comedi_subdevice *s, unsigned int trig_num) { struct usbduxsigma_private *devpriv = dev->private; struct comedi_cmd *cmd = &s->async->cmd; int ret; if (trig_num != cmd->start_arg) return -EINVAL; mutex_lock(&devpriv->mut); if (!devpriv->ai_cmd_running) { devpriv->ai_cmd_running = 1; ret = usbduxsigma_submit_urbs(dev, devpriv->ai_urbs, devpriv->n_ai_urbs, 1); if (ret < 0) { devpriv->ai_cmd_running = 0; mutex_unlock(&devpriv->mut); return ret; } s->async->inttrig = NULL; } mutex_unlock(&devpriv->mut); return 1; } static int usbduxsigma_ai_cmd(struct comedi_device *dev, struct comedi_subdevice *s) { struct usbduxsigma_private *devpriv = dev->private; struct comedi_cmd *cmd = &s->async->cmd; unsigned int len = cmd->chanlist_len; u8 muxsg0 = 0; u8 muxsg1 = 0; u8 sysred = 0; int ret; int i; mutex_lock(&devpriv->mut); if (devpriv->high_speed) { /* * every 2 channels get a time window of 125us. Thus, if we * sample all 16 channels we need 1ms. If we sample only one * channel we need only 125us */ unsigned int interval = usbduxsigma_chans_to_interval(len); devpriv->ai_interval = interval; devpriv->ai_timer = cmd->scan_begin_arg / (125000 * interval); } else { /* interval always 1ms */ devpriv->ai_interval = 1; devpriv->ai_timer = cmd->scan_begin_arg / 1000000; } for (i = 0; i < len; i++) { unsigned int chan = CR_CHAN(cmd->chanlist[i]); create_adc_command(chan, &muxsg0, &muxsg1); } devpriv->dux_commands[1] = devpriv->ai_interval; devpriv->dux_commands[2] = len; /* num channels per time step */ devpriv->dux_commands[3] = 0x12; /* CONFIG0 */ devpriv->dux_commands[4] = 0x03; /* CONFIG1: 23kHz sample, delay 0us */ devpriv->dux_commands[5] = 0x00; /* CONFIG3: diff. channels off */ devpriv->dux_commands[6] = muxsg0; devpriv->dux_commands[7] = muxsg1; devpriv->dux_commands[8] = sysred; ret = usbbuxsigma_send_cmd(dev, USBBUXSIGMA_AD_CMD); if (ret < 0) { mutex_unlock(&devpriv->mut); return ret; } devpriv->ai_counter = devpriv->ai_timer; if (cmd->start_src == TRIG_NOW) { /* enable this acquisition operation */ devpriv->ai_cmd_running = 1; ret = usbduxsigma_submit_urbs(dev, devpriv->ai_urbs, devpriv->n_ai_urbs, 1); if (ret < 0) { devpriv->ai_cmd_running = 0; mutex_unlock(&devpriv->mut); return ret; } s->async->inttrig = NULL; } else { /* TRIG_INT */ s->async->inttrig = usbduxsigma_ai_inttrig; } mutex_unlock(&devpriv->mut); return 0; } static int usbduxsigma_ai_insn_read(struct comedi_device *dev, struct comedi_subdevice *s, struct comedi_insn *insn, unsigned int *data) { struct usbduxsigma_private *devpriv = dev->private; unsigned int chan = CR_CHAN(insn->chanspec); u8 muxsg0 = 0; u8 muxsg1 = 0; u8 sysred = 0; int ret; int i; mutex_lock(&devpriv->mut); if (devpriv->ai_cmd_running) { mutex_unlock(&devpriv->mut); return -EBUSY; } create_adc_command(chan, &muxsg0, &muxsg1); /* Mode 0 is used to get a single conversion on demand */ devpriv->dux_commands[1] = 0x16; /* CONFIG0: chopper on */ devpriv->dux_commands[2] = 0x80; /* CONFIG1: 2kHz sampling rate */ devpriv->dux_commands[3] = 0x00; /* CONFIG3: diff. channels off */ devpriv->dux_commands[4] = muxsg0; devpriv->dux_commands[5] = muxsg1; devpriv->dux_commands[6] = sysred; /* adc commands */ ret = usbbuxsigma_send_cmd(dev, USBDUXSIGMA_SINGLE_AD_CMD); if (ret < 0) { mutex_unlock(&devpriv->mut); return ret; } for (i = 0; i < insn->n; i++) { u32 val; ret = usbduxsigma_receive_cmd(dev, USBDUXSIGMA_SINGLE_AD_CMD); if (ret < 0) { mutex_unlock(&devpriv->mut); return ret; } /* 32 bits big endian from the A/D converter */ val = be32_to_cpu(get_unaligned((__be32 *)(devpriv->insn_buf + 1))); val &= 0x00ffffff; /* strip status byte */ data[i] = comedi_offset_munge(s, val); } mutex_unlock(&devpriv->mut); return insn->n; } static int usbduxsigma_ao_insn_read(struct comedi_device *dev, struct comedi_subdevice *s, struct comedi_insn *insn, unsigned int *data) { struct usbduxsigma_private *devpriv = dev->private; int ret; mutex_lock(&devpriv->mut); ret = comedi_readback_insn_read(dev, s, insn, data); mutex_unlock(&devpriv->mut); return ret; } static int usbduxsigma_ao_insn_write(struct comedi_device *dev, struct comedi_subdevice *s, struct comedi_insn *insn, unsigned int *data) { struct usbduxsigma_private *devpriv = dev->private; unsigned int chan = CR_CHAN(insn->chanspec); int ret; int i; mutex_lock(&devpriv->mut); if (devpriv->ao_cmd_running) { mutex_unlock(&devpriv->mut); return -EBUSY; } for (i = 0; i < insn->n; i++) { devpriv->dux_commands[1] = 1; /* num channels */ devpriv->dux_commands[2] = data[i]; /* value */ devpriv->dux_commands[3] = chan; /* channel number */ ret = usbbuxsigma_send_cmd(dev, USBDUXSIGMA_DA_CMD); if (ret < 0) { mutex_unlock(&devpriv->mut); return ret; } s->readback[chan] = data[i]; } mutex_unlock(&devpriv->mut); return insn->n; } static int usbduxsigma_ao_inttrig(struct comedi_device *dev, struct comedi_subdevice *s, unsigned int trig_num) { struct usbduxsigma_private *devpriv = dev->private; struct comedi_cmd *cmd = &s->async->cmd; int ret; if (trig_num != cmd->start_arg) return -EINVAL; mutex_lock(&devpriv->mut); if (!devpriv->ao_cmd_running) { devpriv->ao_cmd_running = 1; ret = usbduxsigma_submit_urbs(dev, devpriv->ao_urbs, devpriv->n_ao_urbs, 0); if (ret < 0) { devpriv->ao_cmd_running = 0; mutex_unlock(&devpriv->mut); return ret; } s->async->inttrig = NULL; } mutex_unlock(&devpriv->mut); return 1; } static int usbduxsigma_ao_cmdtest(struct comedi_device *dev, struct comedi_subdevice *s, struct comedi_cmd *cmd) { struct usbduxsigma_private *devpriv = dev->private; unsigned int tmp; int err = 0; /* Step 1 : check if triggers are trivially valid */ err |= comedi_check_trigger_src(&cmd->start_src, TRIG_NOW | TRIG_INT); /* * For now, always use "scan" timing with all channels updated at once * (cmd->scan_begin_src == TRIG_TIMER, cmd->convert_src == TRIG_NOW). * * In a future version, "convert" timing with channels updated * indivually may be supported in high speed mode * (cmd->scan_begin_src == TRIG_FOLLOW, cmd->convert_src == TRIG_TIMER). */ err |= comedi_check_trigger_src(&cmd->scan_begin_src, TRIG_TIMER); err |= comedi_check_trigger_src(&cmd->convert_src, TRIG_NOW); err |= comedi_check_trigger_src(&cmd->scan_end_src, TRIG_COUNT); err |= comedi_check_trigger_src(&cmd->stop_src, TRIG_COUNT | TRIG_NONE); if (err) { mutex_unlock(&devpriv->mut); return 1; } /* Step 2a : make sure trigger sources are unique */ err |= comedi_check_trigger_is_unique(cmd->start_src); err |= comedi_check_trigger_is_unique(cmd->stop_src); /* Step 2b : and mutually compatible */ if (err) return 2; /* Step 3: check if arguments are trivially valid */ err |= comedi_check_trigger_arg_is(&cmd->start_arg, 0); err |= comedi_check_trigger_arg_min(&cmd->scan_begin_arg, 1000000); err |= comedi_check_trigger_arg_is(&cmd->scan_end_arg, cmd->chanlist_len); if (cmd->stop_src == TRIG_COUNT) err |= comedi_check_trigger_arg_min(&cmd->stop_arg, 1); else /* TRIG_NONE */ err |= comedi_check_trigger_arg_is(&cmd->stop_arg, 0); if (err) return 3; /* Step 4: fix up any arguments */ tmp = rounddown(cmd->scan_begin_arg, 1000000); err |= comedi_check_trigger_arg_is(&cmd->scan_begin_arg, tmp); if (err) return 4; return 0; } static int usbduxsigma_ao_cmd(struct comedi_device *dev, struct comedi_subdevice *s) { struct usbduxsigma_private *devpriv = dev->private; struct comedi_cmd *cmd = &s->async->cmd; int ret; mutex_lock(&devpriv->mut); /* * For now, only "scan" timing is supported. A future version may * support "convert" timing in high speed mode. * * Timing of the scan: every 1ms all channels updated at once. */ devpriv->ao_timer = cmd->scan_begin_arg / 1000000; devpriv->ao_counter = devpriv->ao_timer; if (cmd->start_src == TRIG_NOW) { /* enable this acquisition operation */ devpriv->ao_cmd_running = 1; ret = usbduxsigma_submit_urbs(dev, devpriv->ao_urbs, devpriv->n_ao_urbs, 0); if (ret < 0) { devpriv->ao_cmd_running = 0; mutex_unlock(&devpriv->mut); return ret; } s->async->inttrig = NULL; } else { /* TRIG_INT */ s->async->inttrig = usbduxsigma_ao_inttrig; } mutex_unlock(&devpriv->mut); return 0; } static int usbduxsigma_dio_insn_config(struct comedi_device *dev, struct comedi_subdevice *s, struct comedi_insn *insn, unsigned int *data) { int ret; ret = comedi_dio_insn_config(dev, s, insn, data, 0); if (ret) return ret; /* * We don't tell the firmware here as it would take 8 frames * to submit the information. We do it in the (*insn_bits). */ return insn->n; } static int usbduxsigma_dio_insn_bits(struct comedi_device *dev, struct comedi_subdevice *s, struct comedi_insn *insn, unsigned int *data) { struct usbduxsigma_private *devpriv = dev->private; int ret; mutex_lock(&devpriv->mut); comedi_dio_update_state(s, data); /* Always update the hardware. See the (*insn_config). */ devpriv->dux_commands[1] = s->io_bits & 0xff; devpriv->dux_commands[4] = s->state & 0xff; devpriv->dux_commands[2] = (s->io_bits >> 8) & 0xff; devpriv->dux_commands[5] = (s->state >> 8) & 0xff; devpriv->dux_commands[3] = (s->io_bits >> 16) & 0xff; devpriv->dux_commands[6] = (s->state >> 16) & 0xff; ret = usbbuxsigma_send_cmd(dev, USBDUXSIGMA_DIO_BITS_CMD); if (ret < 0) goto done; ret = usbduxsigma_receive_cmd(dev, USBDUXSIGMA_DIO_BITS_CMD); if (ret < 0) goto done; s->state = devpriv->insn_buf[1] | (devpriv->insn_buf[2] << 8) | (devpriv->insn_buf[3] << 16); data[1] = s->state; ret = insn->n; done: mutex_unlock(&devpriv->mut); return ret; } static void usbduxsigma_pwm_stop(struct comedi_device *dev, int do_unlink) { struct usbduxsigma_private *devpriv = dev->private; if (do_unlink) { if (devpriv->pwm_urb) usb_kill_urb(devpriv->pwm_urb); } devpriv->pwm_cmd_running = 0; } static int usbduxsigma_pwm_cancel(struct comedi_device *dev, struct comedi_subdevice *s) { struct usbduxsigma_private *devpriv = dev->private; /* unlink only if it is really running */ usbduxsigma_pwm_stop(dev, devpriv->pwm_cmd_running); return usbbuxsigma_send_cmd(dev, USBDUXSIGMA_PWM_OFF_CMD); } static void usbduxsigma_pwm_urb_complete(struct urb *urb) { struct comedi_device *dev = urb->context; struct usbduxsigma_private *devpriv = dev->private; int ret; switch (urb->status) { case 0: /* success */ break; case -ECONNRESET: case -ENOENT: case -ESHUTDOWN: case -ECONNABORTED: /* happens after an unlink command */ if (devpriv->pwm_cmd_running) usbduxsigma_pwm_stop(dev, 0); /* w/o unlink */ return; default: /* a real error */ if (devpriv->pwm_cmd_running) { dev_err(dev->class_dev, "non-zero urb status (%d)\n", urb->status); usbduxsigma_pwm_stop(dev, 0); /* w/o unlink */ } return; } if (!devpriv->pwm_cmd_running) return; urb->transfer_buffer_length = devpriv->pwm_buf_sz; urb->dev = comedi_to_usb_dev(dev); urb->status = 0; ret = usb_submit_urb(urb, GFP_ATOMIC); if (ret < 0) { dev_err(dev->class_dev, "urb resubmit failed (%d)\n", ret); if (ret == -EL2NSYNC) dev_err(dev->class_dev, "buggy USB host controller or bug in IRQ handler\n"); usbduxsigma_pwm_stop(dev, 0); /* w/o unlink */ } } static int usbduxsigma_submit_pwm_urb(struct comedi_device *dev) { struct usb_device *usb = comedi_to_usb_dev(dev); struct usbduxsigma_private *devpriv = dev->private; struct urb *urb = devpriv->pwm_urb; /* in case of a resubmission after an unlink... */ usb_fill_bulk_urb(urb, usb, usb_sndbulkpipe(usb, 4), urb->transfer_buffer, devpriv->pwm_buf_sz, usbduxsigma_pwm_urb_complete, dev); return usb_submit_urb(urb, GFP_ATOMIC); } static int usbduxsigma_pwm_period(struct comedi_device *dev, struct comedi_subdevice *s, unsigned int period) { struct usbduxsigma_private *devpriv = dev->private; int fx2delay; if (period < MIN_PWM_PERIOD) return -EAGAIN; fx2delay = (period / (6 * 512 * 1000 / 33)) - 6; if (fx2delay > 255) return -EAGAIN; devpriv->pwm_delay = fx2delay; devpriv->pwm_period = period; return 0; } static int usbduxsigma_pwm_start(struct comedi_device *dev, struct comedi_subdevice *s) { struct usbduxsigma_private *devpriv = dev->private; int ret; if (devpriv->pwm_cmd_running) return 0; devpriv->dux_commands[1] = devpriv->pwm_delay; ret = usbbuxsigma_send_cmd(dev, USBDUXSIGMA_PWM_ON_CMD); if (ret < 0) return ret; memset(devpriv->pwm_urb->transfer_buffer, 0, devpriv->pwm_buf_sz); devpriv->pwm_cmd_running = 1; ret = usbduxsigma_submit_pwm_urb(dev); if (ret < 0) { devpriv->pwm_cmd_running = 0; return ret; } return 0; } static void usbduxsigma_pwm_pattern(struct comedi_device *dev, struct comedi_subdevice *s, unsigned int chan, unsigned int value, unsigned int sign) { struct usbduxsigma_private *devpriv = dev->private; char pwm_mask = (1 << chan); /* DIO bit for the PWM data */ char sgn_mask = (16 << chan); /* DIO bit for the sign */ char *buf = (char *)(devpriv->pwm_urb->transfer_buffer); int szbuf = devpriv->pwm_buf_sz; int i; for (i = 0; i < szbuf; i++) { char c = *buf; c &= ~pwm_mask; if (i < value) c |= pwm_mask; if (!sign) c &= ~sgn_mask; else c |= sgn_mask; *buf++ = c; } } static int usbduxsigma_pwm_write(struct comedi_device *dev, struct comedi_subdevice *s, struct comedi_insn *insn, unsigned int *data) { unsigned int chan = CR_CHAN(insn->chanspec); /* * It doesn't make sense to support more than one value here * because it would just overwrite the PWM buffer. */ if (insn->n != 1) return -EINVAL; /* * The sign is set via a special INSN only, this gives us 8 bits * for normal operation, sign is 0 by default. */ usbduxsigma_pwm_pattern(dev, s, chan, data[0], 0); return insn->n; } static int usbduxsigma_pwm_config(struct comedi_device *dev, struct comedi_subdevice *s, struct comedi_insn *insn, unsigned int *data) { struct usbduxsigma_private *devpriv = dev->private; unsigned int chan = CR_CHAN(insn->chanspec); switch (data[0]) { case INSN_CONFIG_ARM: /* * if not zero the PWM is limited to a certain time which is * not supported here */ if (data[1] != 0) return -EINVAL; return usbduxsigma_pwm_start(dev, s); case INSN_CONFIG_DISARM: return usbduxsigma_pwm_cancel(dev, s); case INSN_CONFIG_GET_PWM_STATUS: data[1] = devpriv->pwm_cmd_running; return 0; case INSN_CONFIG_PWM_SET_PERIOD: return usbduxsigma_pwm_period(dev, s, data[1]); case INSN_CONFIG_PWM_GET_PERIOD: data[1] = devpriv->pwm_period; return 0; case INSN_CONFIG_PWM_SET_H_BRIDGE: /* * data[1] = value * data[2] = sign (for a relay) */ usbduxsigma_pwm_pattern(dev, s, chan, data[1], (data[2] != 0)); return 0; case INSN_CONFIG_PWM_GET_H_BRIDGE: /* values are not kept in this driver, nothing to return */ return -EINVAL; } return -EINVAL; } static int usbduxsigma_getstatusinfo(struct comedi_device *dev, int chan) { struct comedi_subdevice *s = dev->read_subdev; struct usbduxsigma_private *devpriv = dev->private; u8 sysred; u32 val; int ret; switch (chan) { default: case 0: sysred = 0; /* ADC zero */ break; case 1: sysred = 1; /* ADC offset */ break; case 2: sysred = 4; /* VCC */ break; case 3: sysred = 8; /* temperature */ break; case 4: sysred = 16; /* gain */ break; case 5: sysred = 32; /* ref */ break; } devpriv->dux_commands[1] = 0x12; /* CONFIG0 */ devpriv->dux_commands[2] = 0x80; /* CONFIG1: 2kHz sampling rate */ devpriv->dux_commands[3] = 0x00; /* CONFIG3: diff. channels off */ devpriv->dux_commands[4] = 0; devpriv->dux_commands[5] = 0; devpriv->dux_commands[6] = sysred; ret = usbbuxsigma_send_cmd(dev, USBDUXSIGMA_SINGLE_AD_CMD); if (ret < 0) return ret; ret = usbduxsigma_receive_cmd(dev, USBDUXSIGMA_SINGLE_AD_CMD); if (ret < 0) return ret; /* 32 bits big endian from the A/D converter */ val = be32_to_cpu(get_unaligned((__be32 *)(devpriv->insn_buf + 1))); val &= 0x00ffffff; /* strip status byte */ return (int)comedi_offset_munge(s, val); } static int usbduxsigma_firmware_upload(struct comedi_device *dev, const u8 *data, size_t size, unsigned long context) { struct usb_device *usb = comedi_to_usb_dev(dev); u8 *buf; u8 *tmp; int ret; if (!data) return 0; if (size > FIRMWARE_MAX_LEN) { dev_err(dev->class_dev, "firmware binary too large for FX2\n"); return -ENOMEM; } /* we generate a local buffer for the firmware */ buf = kmemdup(data, size, GFP_KERNEL); if (!buf) return -ENOMEM; /* we need a malloc'ed buffer for usb_control_msg() */ tmp = kmalloc(1, GFP_KERNEL); if (!tmp) { kfree(buf); return -ENOMEM; } /* stop the current firmware on the device */ *tmp = 1; /* 7f92 to one */ ret = usb_control_msg(usb, usb_sndctrlpipe(usb, 0), USBDUXSUB_FIRMWARE, VENDOR_DIR_OUT, USBDUXSUB_CPUCS, 0x0000, tmp, 1, BULK_TIMEOUT); if (ret < 0) { dev_err(dev->class_dev, "can not stop firmware\n"); goto done; } /* upload the new firmware to the device */ ret = usb_control_msg(usb, usb_sndctrlpipe(usb, 0), USBDUXSUB_FIRMWARE, VENDOR_DIR_OUT, 0, 0x0000, buf, size, BULK_TIMEOUT); if (ret < 0) { dev_err(dev->class_dev, "firmware upload failed\n"); goto done; } /* start the new firmware on the device */ *tmp = 0; /* 7f92 to zero */ ret = usb_control_msg(usb, usb_sndctrlpipe(usb, 0), USBDUXSUB_FIRMWARE, VENDOR_DIR_OUT, USBDUXSUB_CPUCS, 0x0000, tmp, 1, BULK_TIMEOUT); if (ret < 0) dev_err(dev->class_dev, "can not start firmware\n"); done: kfree(tmp); kfree(buf); return ret; } static int usbduxsigma_alloc_usb_buffers(struct comedi_device *dev) { struct usb_device *usb = comedi_to_usb_dev(dev); struct usbduxsigma_private *devpriv = dev->private; struct urb *urb; int i; devpriv->dux_commands = kzalloc(SIZEOFDUXBUFFER, GFP_KERNEL); devpriv->in_buf = kzalloc(SIZEINBUF, GFP_KERNEL); devpriv->insn_buf = kzalloc(SIZEINSNBUF, GFP_KERNEL); devpriv->ai_urbs = kcalloc(devpriv->n_ai_urbs, sizeof(urb), GFP_KERNEL); devpriv->ao_urbs = kcalloc(devpriv->n_ao_urbs, sizeof(urb), GFP_KERNEL); if (!devpriv->dux_commands || !devpriv->in_buf || !devpriv->insn_buf || !devpriv->ai_urbs || !devpriv->ao_urbs) return -ENOMEM; for (i = 0; i < devpriv->n_ai_urbs; i++) { /* one frame: 1ms */ urb = usb_alloc_urb(1, GFP_KERNEL); if (!urb) return -ENOMEM; devpriv->ai_urbs[i] = urb; urb->dev = usb; /* will be filled later with a pointer to the comedi-device */ /* and ONLY then the urb should be submitted */ urb->context = NULL; urb->pipe = usb_rcvisocpipe(usb, 6); urb->transfer_flags = URB_ISO_ASAP; urb->transfer_buffer = kzalloc(SIZEINBUF, GFP_KERNEL); if (!urb->transfer_buffer) return -ENOMEM; urb->complete = usbduxsigma_ai_urb_complete; urb->number_of_packets = 1; urb->transfer_buffer_length = SIZEINBUF; urb->iso_frame_desc[0].offset = 0; urb->iso_frame_desc[0].length = SIZEINBUF; } for (i = 0; i < devpriv->n_ao_urbs; i++) { /* one frame: 1ms */ urb = usb_alloc_urb(1, GFP_KERNEL); if (!urb) return -ENOMEM; devpriv->ao_urbs[i] = urb; urb->dev = usb; /* will be filled later with a pointer to the comedi-device */ /* and ONLY then the urb should be submitted */ urb->context = NULL; urb->pipe = usb_sndisocpipe(usb, 2); urb->transfer_flags = URB_ISO_ASAP; urb->transfer_buffer = kzalloc(SIZEOUTBUF, GFP_KERNEL); if (!urb->transfer_buffer) return -ENOMEM; urb->complete = usbduxsigma_ao_urb_complete; urb->number_of_packets = 1; urb->transfer_buffer_length = SIZEOUTBUF; urb->iso_frame_desc[0].offset = 0; urb->iso_frame_desc[0].length = SIZEOUTBUF; urb->interval = 1; /* (u)frames */ } if (devpriv->pwm_buf_sz) { urb = usb_alloc_urb(0, GFP_KERNEL); if (!urb) return -ENOMEM; devpriv->pwm_urb = urb; urb->transfer_buffer = kzalloc(devpriv->pwm_buf_sz, GFP_KERNEL); if (!urb->transfer_buffer) return -ENOMEM; } return 0; } static void usbduxsigma_free_usb_buffers(struct comedi_device *dev) { struct usbduxsigma_private *devpriv = dev->private; struct urb *urb; int i; urb = devpriv->pwm_urb; if (urb) { kfree(urb->transfer_buffer); usb_free_urb(urb); } if (devpriv->ao_urbs) { for (i = 0; i < devpriv->n_ao_urbs; i++) { urb = devpriv->ao_urbs[i]; if (urb) { kfree(urb->transfer_buffer); usb_free_urb(urb); } } kfree(devpriv->ao_urbs); } if (devpriv->ai_urbs) { for (i = 0; i < devpriv->n_ai_urbs; i++) { urb = devpriv->ai_urbs[i]; if (urb) { kfree(urb->transfer_buffer); usb_free_urb(urb); } } kfree(devpriv->ai_urbs); } kfree(devpriv->insn_buf); kfree(devpriv->in_buf); kfree(devpriv->dux_commands); } static int usbduxsigma_auto_attach(struct comedi_device *dev, unsigned long context_unused) { struct usb_interface *intf = comedi_to_usb_interface(dev); struct usb_device *usb = comedi_to_usb_dev(dev); struct usbduxsigma_private *devpriv; struct comedi_subdevice *s; int offset; int ret; devpriv = comedi_alloc_devpriv(dev, sizeof(*devpriv)); if (!devpriv) return -ENOMEM; mutex_init(&devpriv->mut); usb_set_intfdata(intf, devpriv); devpriv->high_speed = (usb->speed == USB_SPEED_HIGH); if (devpriv->high_speed) { devpriv->n_ai_urbs = NUMOFINBUFFERSHIGH; devpriv->n_ao_urbs = NUMOFOUTBUFFERSHIGH; devpriv->pwm_buf_sz = 512; } else { devpriv->n_ai_urbs = NUMOFINBUFFERSFULL; devpriv->n_ao_urbs = NUMOFOUTBUFFERSFULL; } ret = usbduxsigma_alloc_usb_buffers(dev); if (ret) return ret; /* setting to alternate setting 3: enabling iso ep and bulk ep. */ ret = usb_set_interface(usb, intf->altsetting->desc.bInterfaceNumber, 3); if (ret < 0) { dev_err(dev->class_dev, "could not set alternate setting 3 in high speed\n"); return ret; } ret = comedi_load_firmware(dev, &usb->dev, FIRMWARE, usbduxsigma_firmware_upload, 0); if (ret) return ret; ret = comedi_alloc_subdevices(dev, (devpriv->high_speed) ? 4 : 3); if (ret) return ret; /* Analog Input subdevice */ s = &dev->subdevices[0]; dev->read_subdev = s; s->type = COMEDI_SUBD_AI; s->subdev_flags = SDF_READABLE | SDF_GROUND | SDF_CMD_READ | SDF_LSAMPL; s->n_chan = NUMCHANNELS; s->len_chanlist = NUMCHANNELS; s->maxdata = 0x00ffffff; s->range_table = &usbduxsigma_ai_range; s->insn_read = usbduxsigma_ai_insn_read; s->do_cmdtest = usbduxsigma_ai_cmdtest; s->do_cmd = usbduxsigma_ai_cmd; s->cancel = usbduxsigma_ai_cancel; /* Analog Output subdevice */ s = &dev->subdevices[1]; dev->write_subdev = s; s->type = COMEDI_SUBD_AO; s->subdev_flags = SDF_WRITABLE | SDF_GROUND | SDF_CMD_WRITE; s->n_chan = 4; s->len_chanlist = s->n_chan; s->maxdata = 0x00ff; s->range_table = &range_unipolar2_5; s->insn_write = usbduxsigma_ao_insn_write; s->insn_read = usbduxsigma_ao_insn_read; s->do_cmdtest = usbduxsigma_ao_cmdtest; s->do_cmd = usbduxsigma_ao_cmd; s->cancel = usbduxsigma_ao_cancel; ret = comedi_alloc_subdev_readback(s); if (ret) return ret; /* Digital I/O subdevice */ s = &dev->subdevices[2]; s->type = COMEDI_SUBD_DIO; s->subdev_flags = SDF_READABLE | SDF_WRITABLE; s->n_chan = 24; s->maxdata = 1; s->range_table = &range_digital; s->insn_bits = usbduxsigma_dio_insn_bits; s->insn_config = usbduxsigma_dio_insn_config; if (devpriv->high_speed) { /* Timer / pwm subdevice */ s = &dev->subdevices[3]; s->type = COMEDI_SUBD_PWM; s->subdev_flags = SDF_WRITABLE | SDF_PWM_HBRIDGE; s->n_chan = 8; s->maxdata = devpriv->pwm_buf_sz; s->insn_write = usbduxsigma_pwm_write; s->insn_config = usbduxsigma_pwm_config; usbduxsigma_pwm_period(dev, s, PWM_DEFAULT_PERIOD); } offset = usbduxsigma_getstatusinfo(dev, 0); if (offset < 0) { dev_err(dev->class_dev, "Communication to USBDUXSIGMA failed! Check firmware and cabling.\n"); return offset; } dev_info(dev->class_dev, "ADC_zero = %x\n", offset); return 0; } static void usbduxsigma_detach(struct comedi_device *dev) { struct usb_interface *intf = comedi_to_usb_interface(dev); struct usbduxsigma_private *devpriv = dev->private; usb_set_intfdata(intf, NULL); if (!devpriv) return; mutex_lock(&devpriv->mut); /* force unlink all urbs */ usbduxsigma_ai_stop(dev, 1); usbduxsigma_ao_stop(dev, 1); usbduxsigma_pwm_stop(dev, 1); usbduxsigma_free_usb_buffers(dev); mutex_unlock(&devpriv->mut); mutex_destroy(&devpriv->mut); } static struct comedi_driver usbduxsigma_driver = { .driver_name = "usbduxsigma", .module = THIS_MODULE, .auto_attach = usbduxsigma_auto_attach, .detach = usbduxsigma_detach, }; static int usbduxsigma_usb_probe(struct usb_interface *intf, const struct usb_device_id *id) { return comedi_usb_auto_config(intf, &usbduxsigma_driver, 0); } static const struct usb_device_id usbduxsigma_usb_table[] = { { USB_DEVICE(0x13d8, 0x0020) }, { USB_DEVICE(0x13d8, 0x0021) }, { USB_DEVICE(0x13d8, 0x0022) }, { } }; MODULE_DEVICE_TABLE(usb, usbduxsigma_usb_table); static struct usb_driver usbduxsigma_usb_driver = { .name = "usbduxsigma", .probe = usbduxsigma_usb_probe, .disconnect = comedi_usb_auto_unconfig, .id_table = usbduxsigma_usb_table, }; module_comedi_usb_driver(usbduxsigma_driver, usbduxsigma_usb_driver); MODULE_AUTHOR("Bernd Porr, mail@berndporr.me.uk"); MODULE_DESCRIPTION("Stirling/ITL USB-DUX SIGMA -- mail@berndporr.me.uk"); MODULE_LICENSE("GPL"); MODULE_FIRMWARE(FIRMWARE); |
| 5 3 1612 383 383 374 2946 374 2950 643 644 79 79 373 14 2008 2018 2022 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 | // SPDX-License-Identifier: GPL-2.0-only /* * fs/anon_inodes.c * * Copyright (C) 2007 Davide Libenzi <davidel@xmailserver.org> * * Thanks to Arnd Bergmann for code review and suggestions. * More changes for Thomas Gleixner suggestions. * */ #include <linux/cred.h> #include <linux/file.h> #include <linux/poll.h> #include <linux/sched.h> #include <linux/init.h> #include <linux/fs.h> #include <linux/mount.h> #include <linux/module.h> #include <linux/kernel.h> #include <linux/magic.h> #include <linux/anon_inodes.h> #include <linux/pseudo_fs.h> #include <linux/uaccess.h> #include "internal.h" static struct vfsmount *anon_inode_mnt __ro_after_init; static struct inode *anon_inode_inode __ro_after_init; /* * User space expects anonymous inodes to have no file type in st_mode. * * In particular, 'lsof' has this legacy logic: * * type = s->st_mode & S_IFMT; * switch (type) { * ... * case 0: * if (!strcmp(p, "anon_inode")) * Lf->ntype = Ntype = N_ANON_INODE; * * to detect our old anon_inode logic. * * Rather than mess with our internal sane inode data, just fix it * up here in getattr() by masking off the format bits. */ int anon_inode_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { struct inode *inode = d_inode(path->dentry); generic_fillattr(&nop_mnt_idmap, request_mask, inode, stat); stat->mode &= ~S_IFMT; return 0; } int anon_inode_setattr(struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { return -EOPNOTSUPP; } static const struct inode_operations anon_inode_operations = { .getattr = anon_inode_getattr, .setattr = anon_inode_setattr, }; /* * anon_inodefs_dname() is called from d_path(). */ static char *anon_inodefs_dname(struct dentry *dentry, char *buffer, int buflen) { return dynamic_dname(buffer, buflen, "anon_inode:%s", dentry->d_name.name); } static const struct dentry_operations anon_inodefs_dentry_operations = { .d_dname = anon_inodefs_dname, }; static int anon_inodefs_init_fs_context(struct fs_context *fc) { struct pseudo_fs_context *ctx = init_pseudo(fc, ANON_INODE_FS_MAGIC); if (!ctx) return -ENOMEM; fc->s_iflags |= SB_I_NOEXEC; fc->s_iflags |= SB_I_NODEV; ctx->dops = &anon_inodefs_dentry_operations; return 0; } static struct file_system_type anon_inode_fs_type = { .name = "anon_inodefs", .init_fs_context = anon_inodefs_init_fs_context, .kill_sb = kill_anon_super, }; /** * anon_inode_make_secure_inode - allocate an anonymous inode with security context * @sb: [in] Superblock to allocate from * @name: [in] Name of the class of the newfile (e.g., "secretmem") * @context_inode: * [in] Optional parent inode for security inheritance * * The function ensures proper security initialization through the LSM hook * security_inode_init_security_anon(). * * Return: Pointer to new inode on success, ERR_PTR on failure. */ struct inode *anon_inode_make_secure_inode(struct super_block *sb, const char *name, const struct inode *context_inode) { struct inode *inode; int error; inode = alloc_anon_inode(sb); if (IS_ERR(inode)) return inode; inode->i_flags &= ~S_PRIVATE; inode->i_op = &anon_inode_operations; error = security_inode_init_security_anon(inode, &QSTR(name), context_inode); if (error) { iput(inode); return ERR_PTR(error); } return inode; } EXPORT_SYMBOL_FOR_MODULES(anon_inode_make_secure_inode, "kvm"); static struct file *__anon_inode_getfile(const char *name, const struct file_operations *fops, void *priv, int flags, const struct inode *context_inode, bool make_inode) { struct inode *inode; struct file *file; if (fops->owner && !try_module_get(fops->owner)) return ERR_PTR(-ENOENT); if (make_inode) { inode = anon_inode_make_secure_inode(anon_inode_mnt->mnt_sb, name, context_inode); if (IS_ERR(inode)) { file = ERR_CAST(inode); goto err; } } else { inode = anon_inode_inode; if (IS_ERR(inode)) { file = ERR_PTR(-ENODEV); goto err; } /* * We know the anon_inode inode count is always * greater than zero, so ihold() is safe. */ ihold(inode); } file = alloc_file_pseudo(inode, anon_inode_mnt, name, flags & (O_ACCMODE | O_NONBLOCK), fops); if (IS_ERR(file)) goto err_iput; file->f_mapping = inode->i_mapping; file->private_data = priv; return file; err_iput: iput(inode); err: module_put(fops->owner); return file; } /** * anon_inode_getfile - creates a new file instance by hooking it up to an * anonymous inode, and a dentry that describe the "class" * of the file * * @name: [in] name of the "class" of the new file * @fops: [in] file operations for the new file * @priv: [in] private data for the new file (will be file's private_data) * @flags: [in] flags * * Creates a new file by hooking it on a single inode. This is useful for files * that do not need to have a full-fledged inode in order to operate correctly. * All the files created with anon_inode_getfile() will share a single inode, * hence saving memory and avoiding code duplication for the file/inode/dentry * setup. Returns the newly created file* or an error pointer. */ struct file *anon_inode_getfile(const char *name, const struct file_operations *fops, void *priv, int flags) { return __anon_inode_getfile(name, fops, priv, flags, NULL, false); } EXPORT_SYMBOL_GPL(anon_inode_getfile); /** * anon_inode_getfile_fmode - creates a new file instance by hooking it up to an * anonymous inode, and a dentry that describe the "class" * of the file * * @name: [in] name of the "class" of the new file * @fops: [in] file operations for the new file * @priv: [in] private data for the new file (will be file's private_data) * @flags: [in] flags * @f_mode: [in] fmode * * Creates a new file by hooking it on a single inode. This is useful for files * that do not need to have a full-fledged inode in order to operate correctly. * All the files created with anon_inode_getfile() will share a single inode, * hence saving memory and avoiding code duplication for the file/inode/dentry * setup. Allows setting the fmode. Returns the newly created file* or an error * pointer. */ struct file *anon_inode_getfile_fmode(const char *name, const struct file_operations *fops, void *priv, int flags, fmode_t f_mode) { struct file *file; file = __anon_inode_getfile(name, fops, priv, flags, NULL, false); if (!IS_ERR(file)) file->f_mode |= f_mode; return file; } EXPORT_SYMBOL_GPL(anon_inode_getfile_fmode); /** * anon_inode_create_getfile - Like anon_inode_getfile(), but creates a new * !S_PRIVATE anon inode rather than reuse the * singleton anon inode and calls the * inode_init_security_anon() LSM hook. * * @name: [in] name of the "class" of the new file * @fops: [in] file operations for the new file * @priv: [in] private data for the new file (will be file's private_data) * @flags: [in] flags * @context_inode: * [in] the logical relationship with the new inode (optional) * * Create a new anonymous inode and file pair. This can be done for two * reasons: * * - for the inode to have its own security context, so that LSMs can enforce * policy on the inode's creation; * * - if the caller needs a unique inode, for example in order to customize * the size returned by fstat() * * The LSM may use @context_inode in inode_init_security_anon(), but a * reference to it is not held. * * Returns the newly created file* or an error pointer. */ struct file *anon_inode_create_getfile(const char *name, const struct file_operations *fops, void *priv, int flags, const struct inode *context_inode) { return __anon_inode_getfile(name, fops, priv, flags, context_inode, true); } EXPORT_SYMBOL_GPL(anon_inode_create_getfile); static int __anon_inode_getfd(const char *name, const struct file_operations *fops, void *priv, int flags, const struct inode *context_inode, bool make_inode) { int error, fd; struct file *file; error = get_unused_fd_flags(flags); if (error < 0) return error; fd = error; file = __anon_inode_getfile(name, fops, priv, flags, context_inode, make_inode); if (IS_ERR(file)) { error = PTR_ERR(file); goto err_put_unused_fd; } fd_install(fd, file); return fd; err_put_unused_fd: put_unused_fd(fd); return error; } /** * anon_inode_getfd - creates a new file instance by hooking it up to * an anonymous inode and a dentry that describe * the "class" of the file * * @name: [in] name of the "class" of the new file * @fops: [in] file operations for the new file * @priv: [in] private data for the new file (will be file's private_data) * @flags: [in] flags * * Creates a new file by hooking it on a single inode. This is * useful for files that do not need to have a full-fledged inode in * order to operate correctly. All the files created with * anon_inode_getfd() will use the same singleton inode, reducing * memory use and avoiding code duplication for the file/inode/dentry * setup. Returns a newly created file descriptor or an error code. */ int anon_inode_getfd(const char *name, const struct file_operations *fops, void *priv, int flags) { return __anon_inode_getfd(name, fops, priv, flags, NULL, false); } EXPORT_SYMBOL_GPL(anon_inode_getfd); /** * anon_inode_create_getfd - Like anon_inode_getfd(), but creates a new * !S_PRIVATE anon inode rather than reuse the singleton anon inode, and calls * the inode_init_security_anon() LSM hook. * * @name: [in] name of the "class" of the new file * @fops: [in] file operations for the new file * @priv: [in] private data for the new file (will be file's private_data) * @flags: [in] flags * @context_inode: * [in] the logical relationship with the new inode (optional) * * Create a new anonymous inode and file pair. This can be done for two * reasons: * * - for the inode to have its own security context, so that LSMs can enforce * policy on the inode's creation; * * - if the caller needs a unique inode, for example in order to customize * the size returned by fstat() * * The LSM may use @context_inode in inode_init_security_anon(), but a * reference to it is not held. * * Returns a newly created file descriptor or an error code. */ int anon_inode_create_getfd(const char *name, const struct file_operations *fops, void *priv, int flags, const struct inode *context_inode) { return __anon_inode_getfd(name, fops, priv, flags, context_inode, true); } static int __init anon_inode_init(void) { anon_inode_mnt = kern_mount(&anon_inode_fs_type); if (IS_ERR(anon_inode_mnt)) panic("anon_inode_init() kernel mount failed (%ld)\n", PTR_ERR(anon_inode_mnt)); anon_inode_inode = alloc_anon_inode(anon_inode_mnt->mnt_sb); if (IS_ERR(anon_inode_inode)) panic("anon_inode_init() inode allocation failed (%ld)\n", PTR_ERR(anon_inode_inode)); anon_inode_inode->i_op = &anon_inode_operations; return 0; } fs_initcall(anon_inode_init); |
| 30 12 25 3 3 42 93 44 1 34 48 7 4 25 1 1 11 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 | // SPDX-License-Identifier: GPL-2.0-only /* * linux/fs/binfmt_elf.c * * These are the functions used to load ELF format executables as used * on SVr4 machines. Information on the format may be found in the book * "UNIX SYSTEM V RELEASE 4 Programmers Guide: Ansi C and Programming Support * Tools". * * Copyright 1993, 1994: Eric Youngdale (ericy@cais.com). */ #include <linux/module.h> #include <linux/kernel.h> #include <linux/fs.h> #include <linux/log2.h> #include <linux/mm.h> #include <linux/mman.h> #include <linux/errno.h> #include <linux/signal.h> #include <linux/binfmts.h> #include <linux/string.h> #include <linux/file.h> #include <linux/slab.h> #include <linux/personality.h> #include <linux/elfcore.h> #include <linux/init.h> #include <linux/highuid.h> #include <linux/compiler.h> #include <linux/highmem.h> #include <linux/hugetlb.h> #include <linux/pagemap.h> #include <linux/vmalloc.h> #include <linux/security.h> #include <linux/random.h> #include <linux/elf.h> #include <linux/elf-randomize.h> #include <linux/utsname.h> #include <linux/coredump.h> #include <linux/sched.h> #include <linux/sched/coredump.h> #include <linux/sched/task_stack.h> #include <linux/sched/cputime.h> #include <linux/sizes.h> #include <linux/types.h> #include <linux/cred.h> #include <linux/dax.h> #include <linux/uaccess.h> #include <linux/rseq.h> #include <asm/param.h> #include <asm/page.h> #ifndef ELF_COMPAT #define ELF_COMPAT 0 #endif #ifndef user_long_t #define user_long_t long #endif #ifndef user_siginfo_t #define user_siginfo_t siginfo_t #endif /* That's for binfmt_elf_fdpic to deal with */ #ifndef elf_check_fdpic #define elf_check_fdpic(ex) false #endif static int load_elf_binary(struct linux_binprm *bprm); /* * If we don't support core dumping, then supply a NULL so we * don't even try. */ #ifdef CONFIG_ELF_CORE static int elf_core_dump(struct coredump_params *cprm); #else #define elf_core_dump NULL #endif #if ELF_EXEC_PAGESIZE > PAGE_SIZE #define ELF_MIN_ALIGN ELF_EXEC_PAGESIZE #else #define ELF_MIN_ALIGN PAGE_SIZE #endif #ifndef ELF_CORE_EFLAGS #define ELF_CORE_EFLAGS 0 #endif #define ELF_PAGESTART(_v) ((_v) & ~(int)(ELF_MIN_ALIGN-1)) #define ELF_PAGEOFFSET(_v) ((_v) & (ELF_MIN_ALIGN-1)) #define ELF_PAGEALIGN(_v) (((_v) + ELF_MIN_ALIGN - 1) & ~(ELF_MIN_ALIGN - 1)) static struct linux_binfmt elf_format = { .module = THIS_MODULE, .load_binary = load_elf_binary, #ifdef CONFIG_COREDUMP .core_dump = elf_core_dump, .min_coredump = ELF_EXEC_PAGESIZE, #endif }; #define BAD_ADDR(x) (unlikely((unsigned long)(x) >= TASK_SIZE)) static inline void elf_coredump_set_mm_eflags(struct mm_struct *mm, u32 flags) { #ifdef CONFIG_ARCH_HAS_ELF_CORE_EFLAGS mm->saved_e_flags = flags; #endif } static inline u32 elf_coredump_get_mm_eflags(struct mm_struct *mm, u32 flags) { #ifdef CONFIG_ARCH_HAS_ELF_CORE_EFLAGS flags = mm->saved_e_flags; #endif return flags; } /* * We need to explicitly zero any trailing portion of the page that follows * p_filesz when it ends before the page ends (e.g. bss), otherwise this * memory will contain the junk from the file that should not be present. */ static int padzero(unsigned long address) { unsigned long nbyte; nbyte = ELF_PAGEOFFSET(address); if (nbyte) { nbyte = ELF_MIN_ALIGN - nbyte; if (clear_user((void __user *)address, nbyte)) return -EFAULT; } return 0; } /* Let's use some macros to make this stack manipulation a little clearer */ #ifdef CONFIG_STACK_GROWSUP #define STACK_ADD(sp, items) ((elf_addr_t __user *)(sp) + (items)) #define STACK_ROUND(sp, items) \ ((15 + (unsigned long) ((sp) + (items))) &~ 15UL) #define STACK_ALLOC(sp, len) ({ \ elf_addr_t __user *old_sp = (elf_addr_t __user *)sp; sp += len; \ old_sp; }) #else #define STACK_ADD(sp, items) ((elf_addr_t __user *)(sp) - (items)) #define STACK_ROUND(sp, items) \ (((unsigned long) (sp - items)) &~ 15UL) #define STACK_ALLOC(sp, len) (sp -= len) #endif #ifndef ELF_BASE_PLATFORM /* * AT_BASE_PLATFORM indicates the "real" hardware/microarchitecture. * If the arch defines ELF_BASE_PLATFORM (in asm/elf.h), the value * will be copied to the user stack in the same manner as AT_PLATFORM. */ #define ELF_BASE_PLATFORM NULL #endif static int create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, unsigned long interp_load_addr, unsigned long e_entry, unsigned long phdr_addr) { struct mm_struct *mm = current->mm; unsigned long p = bprm->p; int argc = bprm->argc; int envc = bprm->envc; elf_addr_t __user *sp; elf_addr_t __user *u_platform; elf_addr_t __user *u_base_platform; elf_addr_t __user *u_rand_bytes; const char *k_platform = ELF_PLATFORM; const char *k_base_platform = ELF_BASE_PLATFORM; unsigned char k_rand_bytes[16]; int items; elf_addr_t *elf_info; elf_addr_t flags = 0; int ei_index; const struct cred *cred = current_cred(); struct vm_area_struct *vma; /* * In some cases (e.g. Hyper-Threading), we want to avoid L1 * evictions by the processes running on the same package. One * thing we can do is to shuffle the initial stack for them. */ p = arch_align_stack(p); /* * If this architecture has a platform capability string, copy it * to userspace. In some cases (Sparc), this info is impossible * for userspace to get any other way, in others (i386) it is * merely difficult. */ u_platform = NULL; if (k_platform) { size_t len = strlen(k_platform) + 1; u_platform = (elf_addr_t __user *)STACK_ALLOC(p, len); if (copy_to_user(u_platform, k_platform, len)) return -EFAULT; } /* * If this architecture has a "base" platform capability * string, copy it to userspace. */ u_base_platform = NULL; if (k_base_platform) { size_t len = strlen(k_base_platform) + 1; u_base_platform = (elf_addr_t __user *)STACK_ALLOC(p, len); if (copy_to_user(u_base_platform, k_base_platform, len)) return -EFAULT; } /* * Generate 16 random bytes for userspace PRNG seeding. */ get_random_bytes(k_rand_bytes, sizeof(k_rand_bytes)); u_rand_bytes = (elf_addr_t __user *) STACK_ALLOC(p, sizeof(k_rand_bytes)); if (copy_to_user(u_rand_bytes, k_rand_bytes, sizeof(k_rand_bytes))) return -EFAULT; /* Create the ELF interpreter info */ elf_info = (elf_addr_t *)mm->saved_auxv; /* update AT_VECTOR_SIZE_BASE if the number of NEW_AUX_ENT() changes */ #define NEW_AUX_ENT(id, val) \ do { \ *elf_info++ = id; \ *elf_info++ = val; \ } while (0) #ifdef ARCH_DLINFO /* * ARCH_DLINFO must come first so PPC can do its special alignment of * AUXV. * update AT_VECTOR_SIZE_ARCH if the number of NEW_AUX_ENT() in * ARCH_DLINFO changes */ ARCH_DLINFO; #endif NEW_AUX_ENT(AT_HWCAP, ELF_HWCAP); NEW_AUX_ENT(AT_PAGESZ, ELF_EXEC_PAGESIZE); NEW_AUX_ENT(AT_CLKTCK, CLOCKS_PER_SEC); NEW_AUX_ENT(AT_PHDR, phdr_addr); NEW_AUX_ENT(AT_PHENT, sizeof(struct elf_phdr)); NEW_AUX_ENT(AT_PHNUM, exec->e_phnum); NEW_AUX_ENT(AT_BASE, interp_load_addr); if (bprm->interp_flags & BINPRM_FLAGS_PRESERVE_ARGV0) flags |= AT_FLAGS_PRESERVE_ARGV0; NEW_AUX_ENT(AT_FLAGS, flags); NEW_AUX_ENT(AT_ENTRY, e_entry); NEW_AUX_ENT(AT_UID, from_kuid_munged(cred->user_ns, cred->uid)); NEW_AUX_ENT(AT_EUID, from_kuid_munged(cred->user_ns, cred->euid)); NEW_AUX_ENT(AT_GID, from_kgid_munged(cred->user_ns, cred->gid)); NEW_AUX_ENT(AT_EGID, from_kgid_munged(cred->user_ns, cred->egid)); NEW_AUX_ENT(AT_SECURE, bprm->secureexec); NEW_AUX_ENT(AT_RANDOM, (elf_addr_t)(unsigned long)u_rand_bytes); #ifdef ELF_HWCAP2 NEW_AUX_ENT(AT_HWCAP2, ELF_HWCAP2); #endif #ifdef ELF_HWCAP3 NEW_AUX_ENT(AT_HWCAP3, ELF_HWCAP3); #endif #ifdef ELF_HWCAP4 NEW_AUX_ENT(AT_HWCAP4, ELF_HWCAP4); #endif NEW_AUX_ENT(AT_EXECFN, bprm->exec); if (k_platform) { NEW_AUX_ENT(AT_PLATFORM, (elf_addr_t)(unsigned long)u_platform); } if (k_base_platform) { NEW_AUX_ENT(AT_BASE_PLATFORM, (elf_addr_t)(unsigned long)u_base_platform); } if (bprm->have_execfd) { NEW_AUX_ENT(AT_EXECFD, bprm->execfd); } #ifdef CONFIG_RSEQ NEW_AUX_ENT(AT_RSEQ_FEATURE_SIZE, offsetof(struct rseq, end)); NEW_AUX_ENT(AT_RSEQ_ALIGN, __alignof__(struct rseq)); #endif #undef NEW_AUX_ENT /* AT_NULL is zero; clear the rest too */ memset(elf_info, 0, (char *)mm->saved_auxv + sizeof(mm->saved_auxv) - (char *)elf_info); /* And advance past the AT_NULL entry. */ elf_info += 2; ei_index = elf_info - (elf_addr_t *)mm->saved_auxv; sp = STACK_ADD(p, ei_index); items = (argc + 1) + (envc + 1) + 1; bprm->p = STACK_ROUND(sp, items); /* Point sp at the lowest address on the stack */ #ifdef CONFIG_STACK_GROWSUP sp = (elf_addr_t __user *)bprm->p - items - ei_index; bprm->exec = (unsigned long)sp; /* XXX: PARISC HACK */ #else sp = (elf_addr_t __user *)bprm->p; #endif /* * Grow the stack manually; some architectures have a limit on how * far ahead a user-space access may be in order to grow the stack. */ if (mmap_write_lock_killable(mm)) return -EINTR; vma = find_extend_vma_locked(mm, bprm->p); mmap_write_unlock(mm); if (!vma) return -EFAULT; /* Now, let's put argc (and argv, envp if appropriate) on the stack */ if (put_user(argc, sp++)) return -EFAULT; /* Populate list of argv pointers back to argv strings. */ p = mm->arg_end = mm->arg_start; while (argc-- > 0) { size_t len; if (put_user((elf_addr_t)p, sp++)) return -EFAULT; len = strnlen_user((void __user *)p, MAX_ARG_STRLEN); if (!len || len > MAX_ARG_STRLEN) return -EINVAL; p += len; } if (put_user(0, sp++)) return -EFAULT; mm->arg_end = p; /* Populate list of envp pointers back to envp strings. */ mm->env_end = mm->env_start = p; while (envc-- > 0) { size_t len; if (put_user((elf_addr_t)p, sp++)) return -EFAULT; len = strnlen_user((void __user *)p, MAX_ARG_STRLEN); if (!len || len > MAX_ARG_STRLEN) return -EINVAL; p += len; } if (put_user(0, sp++)) return -EFAULT; mm->env_end = p; /* Put the elf_info on the stack in the right place. */ if (copy_to_user(sp, mm->saved_auxv, ei_index * sizeof(elf_addr_t))) return -EFAULT; return 0; } /* * Map "eppnt->p_filesz" bytes from "filep" offset "eppnt->p_offset" * into memory at "addr". (Note that p_filesz is rounded up to the * next page, so any extra bytes from the file must be wiped.) */ static unsigned long elf_map(struct file *filep, unsigned long addr, const struct elf_phdr *eppnt, int prot, int type, unsigned long total_size) { unsigned long map_addr; unsigned long size = eppnt->p_filesz + ELF_PAGEOFFSET(eppnt->p_vaddr); unsigned long off = eppnt->p_offset - ELF_PAGEOFFSET(eppnt->p_vaddr); addr = ELF_PAGESTART(addr); size = ELF_PAGEALIGN(size); /* mmap() will return -EINVAL if given a zero size, but a * segment with zero filesize is perfectly valid */ if (!size) return addr; /* * total_size is the size of the ELF (interpreter) image. * The _first_ mmap needs to know the full size, otherwise * randomization might put this image into an overlapping * position with the ELF binary image. (since size < total_size) * So we first map the 'big' image - and unmap the remainder at * the end. (which unmap is needed for ELF images with holes.) */ if (total_size) { total_size = ELF_PAGEALIGN(total_size); map_addr = vm_mmap(filep, addr, total_size, prot, type, off); if (!BAD_ADDR(map_addr)) vm_munmap(map_addr+size, total_size-size); } else map_addr = vm_mmap(filep, addr, size, prot, type, off); if ((type & MAP_FIXED_NOREPLACE) && PTR_ERR((void *)map_addr) == -EEXIST) pr_info("%d (%s): Uhuuh, elf segment at %px requested but the memory is mapped already\n", task_pid_nr(current), current->comm, (void *)addr); return(map_addr); } /* * Map "eppnt->p_filesz" bytes from "filep" offset "eppnt->p_offset" * into memory at "addr". Memory from "p_filesz" through "p_memsz" * rounded up to the next page is zeroed. */ static unsigned long elf_load(struct file *filep, unsigned long addr, const struct elf_phdr *eppnt, int prot, int type, unsigned long total_size) { unsigned long zero_start, zero_end; unsigned long map_addr; if (eppnt->p_filesz) { map_addr = elf_map(filep, addr, eppnt, prot, type, total_size); if (BAD_ADDR(map_addr)) return map_addr; if (eppnt->p_memsz > eppnt->p_filesz) { zero_start = map_addr + ELF_PAGEOFFSET(eppnt->p_vaddr) + eppnt->p_filesz; zero_end = map_addr + ELF_PAGEOFFSET(eppnt->p_vaddr) + eppnt->p_memsz; /* * Zero the end of the last mapped page but ignore * any errors if the segment isn't writable. */ if (padzero(zero_start) && (prot & PROT_WRITE)) return -EFAULT; } } else { map_addr = zero_start = ELF_PAGESTART(addr); zero_end = zero_start + ELF_PAGEOFFSET(eppnt->p_vaddr) + eppnt->p_memsz; } if (eppnt->p_memsz > eppnt->p_filesz) { /* * Map the last of the segment. * If the header is requesting these pages to be * executable, honour that (ppc32 needs this). */ int error; zero_start = ELF_PAGEALIGN(zero_start); zero_end = ELF_PAGEALIGN(zero_end); error = vm_brk_flags(zero_start, zero_end - zero_start, prot & PROT_EXEC ? VM_EXEC : 0); if (error) map_addr = error; } return map_addr; } static unsigned long total_mapping_size(const struct elf_phdr *phdr, int nr) { elf_addr_t min_addr = -1; elf_addr_t max_addr = 0; bool pt_load = false; int i; for (i = 0; i < nr; i++) { if (phdr[i].p_type == PT_LOAD) { min_addr = min(min_addr, ELF_PAGESTART(phdr[i].p_vaddr)); max_addr = max(max_addr, phdr[i].p_vaddr + phdr[i].p_memsz); pt_load = true; } } return pt_load ? (max_addr - min_addr) : 0; } static int elf_read(struct file *file, void *buf, size_t len, loff_t pos) { ssize_t rv; rv = kernel_read(file, buf, len, &pos); if (unlikely(rv != len)) { return (rv < 0) ? rv : -EIO; } return 0; } static unsigned long maximum_alignment(struct elf_phdr *cmds, int nr) { unsigned long alignment = 0; int i; for (i = 0; i < nr; i++) { if (cmds[i].p_type == PT_LOAD) { unsigned long p_align = cmds[i].p_align; /* skip non-power of two alignments as invalid */ if (!is_power_of_2(p_align)) continue; alignment = max(alignment, p_align); } } /* ensure we align to at least one page */ return ELF_PAGEALIGN(alignment); } /** * load_elf_phdrs() - load ELF program headers * @elf_ex: ELF header of the binary whose program headers should be loaded * @elf_file: the opened ELF binary file * * Loads ELF program headers from the binary file elf_file, which has the ELF * header pointed to by elf_ex, into a newly allocated array. The caller is * responsible for freeing the allocated data. Returns NULL upon failure. */ static struct elf_phdr *load_elf_phdrs(const struct elfhdr *elf_ex, struct file *elf_file) { struct elf_phdr *elf_phdata = NULL; int retval = -1; unsigned int size; /* * If the size of this structure has changed, then punt, since * we will be doing the wrong thing. */ if (elf_ex->e_phentsize != sizeof(struct elf_phdr)) goto out; /* Sanity check the number of program headers... */ /* ...and their total size. */ size = sizeof(struct elf_phdr) * elf_ex->e_phnum; if (size == 0 || size > 65536) goto out; elf_phdata = kmalloc(size, GFP_KERNEL); if (!elf_phdata) goto out; /* Read in the program headers */ retval = elf_read(elf_file, elf_phdata, size, elf_ex->e_phoff); out: if (retval) { kfree(elf_phdata); elf_phdata = NULL; } return elf_phdata; } #ifndef CONFIG_ARCH_BINFMT_ELF_STATE /** * struct arch_elf_state - arch-specific ELF loading state * * This structure is used to preserve architecture specific data during * the loading of an ELF file, throughout the checking of architecture * specific ELF headers & through to the point where the ELF load is * known to be proceeding (ie. SET_PERSONALITY). * * This implementation is a dummy for architectures which require no * specific state. */ struct arch_elf_state { }; #define INIT_ARCH_ELF_STATE {} /** * arch_elf_pt_proc() - check a PT_LOPROC..PT_HIPROC ELF program header * @ehdr: The main ELF header * @phdr: The program header to check * @elf: The open ELF file * @is_interp: True if the phdr is from the interpreter of the ELF being * loaded, else false. * @state: Architecture-specific state preserved throughout the process * of loading the ELF. * * Inspects the program header phdr to validate its correctness and/or * suitability for the system. Called once per ELF program header in the * range PT_LOPROC to PT_HIPROC, for both the ELF being loaded and its * interpreter. * * Return: Zero to proceed with the ELF load, non-zero to fail the ELF load * with that return code. */ static inline int arch_elf_pt_proc(struct elfhdr *ehdr, struct elf_phdr *phdr, struct file *elf, bool is_interp, struct arch_elf_state *state) { /* Dummy implementation, always proceed */ return 0; } /** * arch_check_elf() - check an ELF executable * @ehdr: The main ELF header * @has_interp: True if the ELF has an interpreter, else false. * @interp_ehdr: The interpreter's ELF header * @state: Architecture-specific state preserved throughout the process * of loading the ELF. * * Provides a final opportunity for architecture code to reject the loading * of the ELF & cause an exec syscall to return an error. This is called after * all program headers to be checked by arch_elf_pt_proc have been. * * Return: Zero to proceed with the ELF load, non-zero to fail the ELF load * with that return code. */ static inline int arch_check_elf(struct elfhdr *ehdr, bool has_interp, struct elfhdr *interp_ehdr, struct arch_elf_state *state) { /* Dummy implementation, always proceed */ return 0; } #endif /* !CONFIG_ARCH_BINFMT_ELF_STATE */ static inline int make_prot(u32 p_flags, struct arch_elf_state *arch_state, bool has_interp, bool is_interp) { int prot = 0; if (p_flags & PF_R) prot |= PROT_READ; if (p_flags & PF_W) prot |= PROT_WRITE; if (p_flags & PF_X) prot |= PROT_EXEC; return arch_elf_adjust_prot(prot, arch_state, has_interp, is_interp); } /* This is much more generalized than the library routine read function, so we keep this separate. Technically the library read function is only provided so that we can read a.out libraries that have an ELF header */ static unsigned long load_elf_interp(struct elfhdr *interp_elf_ex, struct file *interpreter, unsigned long no_base, struct elf_phdr *interp_elf_phdata, struct arch_elf_state *arch_state) { struct elf_phdr *eppnt; unsigned long load_addr = 0; int load_addr_set = 0; unsigned long error = ~0UL; unsigned long total_size; int i; /* First of all, some simple consistency checks */ if (interp_elf_ex->e_type != ET_EXEC && interp_elf_ex->e_type != ET_DYN) goto out; if (!elf_check_arch(interp_elf_ex) || elf_check_fdpic(interp_elf_ex)) goto out; if (!can_mmap_file(interpreter)) goto out; total_size = total_mapping_size(interp_elf_phdata, interp_elf_ex->e_phnum); if (!total_size) { error = -EINVAL; goto out; } eppnt = interp_elf_phdata; for (i = 0; i < interp_elf_ex->e_phnum; i++, eppnt++) { if (eppnt->p_type == PT_LOAD) { int elf_type = MAP_PRIVATE; int elf_prot = make_prot(eppnt->p_flags, arch_state, true, true); unsigned long vaddr = 0; unsigned long k, map_addr; vaddr = eppnt->p_vaddr; if (interp_elf_ex->e_type == ET_EXEC || load_addr_set) elf_type |= MAP_FIXED; else if (no_base && interp_elf_ex->e_type == ET_DYN) load_addr = -vaddr; map_addr = elf_load(interpreter, load_addr + vaddr, eppnt, elf_prot, elf_type, total_size); total_size = 0; error = map_addr; if (BAD_ADDR(map_addr)) goto out; if (!load_addr_set && interp_elf_ex->e_type == ET_DYN) { load_addr = map_addr - ELF_PAGESTART(vaddr); load_addr_set = 1; } /* * Check to see if the section's size will overflow the * allowed task size. Note that p_filesz must always be * <= p_memsize so it's only necessary to check p_memsz. */ k = load_addr + eppnt->p_vaddr; if (BAD_ADDR(k) || eppnt->p_filesz > eppnt->p_memsz || eppnt->p_memsz > TASK_SIZE || TASK_SIZE - eppnt->p_memsz < k) { error = -ENOMEM; goto out; } } } error = load_addr; out: return error; } /* * These are the functions used to load ELF style executables and shared * libraries. There is no binary dependent code anywhere else. */ static int parse_elf_property(const char *data, size_t *off, size_t datasz, struct arch_elf_state *arch, bool have_prev_type, u32 *prev_type) { size_t o, step; const struct gnu_property *pr; int ret; if (*off == datasz) return -ENOENT; if (WARN_ON_ONCE(*off > datasz || *off % ELF_GNU_PROPERTY_ALIGN)) return -EIO; o = *off; datasz -= *off; if (datasz < sizeof(*pr)) return -ENOEXEC; pr = (const struct gnu_property *)(data + o); o += sizeof(*pr); datasz -= sizeof(*pr); if (pr->pr_datasz > datasz) return -ENOEXEC; WARN_ON_ONCE(o % ELF_GNU_PROPERTY_ALIGN); step = round_up(pr->pr_datasz, ELF_GNU_PROPERTY_ALIGN); if (step > datasz) return -ENOEXEC; /* Properties are supposed to be unique and sorted on pr_type: */ if (have_prev_type && pr->pr_type <= *prev_type) return -ENOEXEC; *prev_type = pr->pr_type; ret = arch_parse_elf_property(pr->pr_type, data + o, pr->pr_datasz, ELF_COMPAT, arch); if (ret) return ret; *off = o + step; return 0; } #define NOTE_DATA_SZ SZ_1K #define NOTE_NAME_SZ (sizeof(NN_GNU_PROPERTY_TYPE_0)) static int parse_elf_properties(struct file *f, const struct elf_phdr *phdr, struct arch_elf_state *arch) { union { struct elf_note nhdr; char data[NOTE_DATA_SZ]; } note; loff_t pos; ssize_t n; size_t off, datasz; int ret; bool have_prev_type; u32 prev_type; if (!IS_ENABLED(CONFIG_ARCH_USE_GNU_PROPERTY) || !phdr) return 0; /* load_elf_binary() shouldn't call us unless this is true... */ if (WARN_ON_ONCE(phdr->p_type != PT_GNU_PROPERTY)) return -ENOEXEC; /* If the properties are crazy large, that's too bad (for now): */ if (phdr->p_filesz > sizeof(note)) return -ENOEXEC; pos = phdr->p_offset; n = kernel_read(f, ¬e, phdr->p_filesz, &pos); BUILD_BUG_ON(sizeof(note) < sizeof(note.nhdr) + NOTE_NAME_SZ); if (n < 0 || n < sizeof(note.nhdr) + NOTE_NAME_SZ) return -EIO; if (note.nhdr.n_type != NT_GNU_PROPERTY_TYPE_0 || note.nhdr.n_namesz != NOTE_NAME_SZ || strncmp(note.data + sizeof(note.nhdr), NN_GNU_PROPERTY_TYPE_0, n - sizeof(note.nhdr))) return -ENOEXEC; off = round_up(sizeof(note.nhdr) + NOTE_NAME_SZ, ELF_GNU_PROPERTY_ALIGN); if (off > n) return -ENOEXEC; if (note.nhdr.n_descsz > n - off) return -ENOEXEC; datasz = off + note.nhdr.n_descsz; have_prev_type = false; do { ret = parse_elf_property(note.data, &off, datasz, arch, have_prev_type, &prev_type); have_prev_type = true; } while (!ret); return ret == -ENOENT ? 0 : ret; } static int load_elf_binary(struct linux_binprm *bprm) { struct file *interpreter = NULL; /* to shut gcc up */ unsigned long load_bias = 0, phdr_addr = 0; int first_pt_load = 1; unsigned long error; struct elf_phdr *elf_ppnt, *elf_phdata, *interp_elf_phdata = NULL; struct elf_phdr *elf_property_phdata = NULL; unsigned long elf_brk; bool brk_moved = false; int retval, i; unsigned long elf_entry; unsigned long e_entry; unsigned long interp_load_addr = 0; unsigned long start_code, end_code, start_data, end_data; unsigned long reloc_func_desc __maybe_unused = 0; int executable_stack = EXSTACK_DEFAULT; struct elfhdr *elf_ex = (struct elfhdr *)bprm->buf; struct elfhdr *interp_elf_ex = NULL; struct arch_elf_state arch_state = INIT_ARCH_ELF_STATE; struct mm_struct *mm; struct pt_regs *regs; retval = -ENOEXEC; /* First of all, some simple consistency checks */ if (memcmp(elf_ex->e_ident, ELFMAG, SELFMAG) != 0) goto out; if (elf_ex->e_type != ET_EXEC && elf_ex->e_type != ET_DYN) goto out; if (!elf_check_arch(elf_ex)) goto out; if (elf_check_fdpic(elf_ex)) goto out; if (!can_mmap_file(bprm->file)) goto out; elf_phdata = load_elf_phdrs(elf_ex, bprm->file); if (!elf_phdata) goto out; elf_ppnt = elf_phdata; for (i = 0; i < elf_ex->e_phnum; i++, elf_ppnt++) { char *elf_interpreter; if (elf_ppnt->p_type == PT_GNU_PROPERTY) { elf_property_phdata = elf_ppnt; continue; } if (elf_ppnt->p_type != PT_INTERP) continue; /* * This is the program interpreter used for shared libraries - * for now assume that this is an a.out format binary. */ retval = -ENOEXEC; if (elf_ppnt->p_filesz > PATH_MAX || elf_ppnt->p_filesz < 2) goto out_free_ph; retval = -ENOMEM; elf_interpreter = kmalloc(elf_ppnt->p_filesz, GFP_KERNEL); if (!elf_interpreter) goto out_free_ph; retval = elf_read(bprm->file, elf_interpreter, elf_ppnt->p_filesz, elf_ppnt->p_offset); if (retval < 0) goto out_free_interp; /* make sure path is NULL terminated */ retval = -ENOEXEC; if (elf_interpreter[elf_ppnt->p_filesz - 1] != '\0') goto out_free_interp; interpreter = open_exec(elf_interpreter); kfree(elf_interpreter); retval = PTR_ERR(interpreter); if (IS_ERR(interpreter)) goto out_free_ph; /* * If the binary is not readable then enforce mm->dumpable = 0 * regardless of the interpreter's permissions. */ would_dump(bprm, interpreter); interp_elf_ex = kmalloc(sizeof(*interp_elf_ex), GFP_KERNEL); if (!interp_elf_ex) { retval = -ENOMEM; goto out_free_file; } /* Get the exec headers */ retval = elf_read(interpreter, interp_elf_ex, sizeof(*interp_elf_ex), 0); if (retval < 0) goto out_free_dentry; break; out_free_interp: kfree(elf_interpreter); goto out_free_ph; } elf_ppnt = elf_phdata; for (i = 0; i < elf_ex->e_phnum; i++, elf_ppnt++) switch (elf_ppnt->p_type) { case PT_GNU_STACK: if (elf_ppnt->p_flags & PF_X) executable_stack = EXSTACK_ENABLE_X; else executable_stack = EXSTACK_DISABLE_X; break; case PT_LOPROC ... PT_HIPROC: retval = arch_elf_pt_proc(elf_ex, elf_ppnt, bprm->file, false, &arch_state); if (retval) goto out_free_dentry; break; } /* Some simple consistency checks for the interpreter */ if (interpreter) { retval = -ELIBBAD; /* Not an ELF interpreter */ if (memcmp(interp_elf_ex->e_ident, ELFMAG, SELFMAG) != 0) goto out_free_dentry; /* Verify the interpreter has a valid arch */ if (!elf_check_arch(interp_elf_ex) || elf_check_fdpic(interp_elf_ex)) goto out_free_dentry; /* Load the interpreter program headers */ interp_elf_phdata = load_elf_phdrs(interp_elf_ex, interpreter); if (!interp_elf_phdata) goto out_free_dentry; /* Pass PT_LOPROC..PT_HIPROC headers to arch code */ elf_property_phdata = NULL; elf_ppnt = interp_elf_phdata; for (i = 0; i < interp_elf_ex->e_phnum; i++, elf_ppnt++) switch (elf_ppnt->p_type) { case PT_GNU_PROPERTY: elf_property_phdata = elf_ppnt; break; case PT_LOPROC ... PT_HIPROC: retval = arch_elf_pt_proc(interp_elf_ex, elf_ppnt, interpreter, true, &arch_state); if (retval) goto out_free_dentry; break; } } retval = parse_elf_properties(interpreter ?: bprm->file, elf_property_phdata, &arch_state); if (retval) goto out_free_dentry; /* * Allow arch code to reject the ELF at this point, whilst it's * still possible to return an error to the code that invoked * the exec syscall. */ retval = arch_check_elf(elf_ex, !!interpreter, interp_elf_ex, &arch_state); if (retval) goto out_free_dentry; /* Flush all traces of the currently running executable */ retval = begin_new_exec(bprm); if (retval) goto out_free_dentry; /* Do this immediately, since STACK_TOP as used in setup_arg_pages may depend on the personality. */ SET_PERSONALITY2(*elf_ex, &arch_state); if (elf_read_implies_exec(*elf_ex, executable_stack)) current->personality |= READ_IMPLIES_EXEC; const int snapshot_randomize_va_space = READ_ONCE(randomize_va_space); if (!(current->personality & ADDR_NO_RANDOMIZE) && snapshot_randomize_va_space) current->flags |= PF_RANDOMIZE; setup_new_exec(bprm); /* Do this so that we can load the interpreter, if need be. We will change some of these later */ retval = setup_arg_pages(bprm, randomize_stack_top(STACK_TOP), executable_stack); if (retval < 0) goto out_free_dentry; elf_brk = 0; start_code = ~0UL; end_code = 0; start_data = 0; end_data = 0; /* Now we do a little grungy work by mmapping the ELF image into the correct location in memory. */ for(i = 0, elf_ppnt = elf_phdata; i < elf_ex->e_phnum; i++, elf_ppnt++) { int elf_prot, elf_flags; unsigned long k, vaddr; unsigned long total_size = 0; unsigned long alignment; if (elf_ppnt->p_type != PT_LOAD) continue; elf_prot = make_prot(elf_ppnt->p_flags, &arch_state, !!interpreter, false); elf_flags = MAP_PRIVATE; vaddr = elf_ppnt->p_vaddr; /* * The first time through the loop, first_pt_load is true: * layout will be calculated. Once set, use MAP_FIXED since * we know we've already safely mapped the entire region with * MAP_FIXED_NOREPLACE in the once-per-binary logic following. */ if (!first_pt_load) { elf_flags |= MAP_FIXED; } else if (elf_ex->e_type == ET_EXEC) { /* * This logic is run once for the first LOAD Program * Header for ET_EXEC binaries. No special handling * is needed. */ elf_flags |= MAP_FIXED_NOREPLACE; } else if (elf_ex->e_type == ET_DYN) { /* * This logic is run once for the first LOAD Program * Header for ET_DYN binaries to calculate the * randomization (load_bias) for all the LOAD * Program Headers. */ /* * Calculate the entire size of the ELF mapping * (total_size), used for the initial mapping, * due to load_addr_set which is set to true later * once the initial mapping is performed. * * Note that this is only sensible when the LOAD * segments are contiguous (or overlapping). If * used for LOADs that are far apart, this would * cause the holes between LOADs to be mapped, * running the risk of having the mapping fail, * as it would be larger than the ELF file itself. * * As a result, only ET_DYN does this, since * some ET_EXEC (e.g. ia64) may have large virtual * memory holes between LOADs. * */ total_size = total_mapping_size(elf_phdata, elf_ex->e_phnum); if (!total_size) { retval = -EINVAL; goto out_free_dentry; } /* Calculate any requested alignment. */ alignment = maximum_alignment(elf_phdata, elf_ex->e_phnum); /** * DOC: PIE handling * * There are effectively two types of ET_DYN ELF * binaries: programs (i.e. PIE: ET_DYN with * PT_INTERP) and loaders (i.e. static PIE: ET_DYN * without PT_INTERP, usually the ELF interpreter * itself). Loaders must be loaded away from programs * since the program may otherwise collide with the * loader (especially for ET_EXEC which does not have * a randomized position). * * For example, to handle invocations of * "./ld.so someprog" to test out a new version of * the loader, the subsequent program that the * loader loads must avoid the loader itself, so * they cannot share the same load range. Sufficient * room for the brk must be allocated with the * loader as well, since brk must be available with * the loader. * * Therefore, programs are loaded offset from * ELF_ET_DYN_BASE and loaders are loaded into the * independently randomized mmap region (0 load_bias * without MAP_FIXED nor MAP_FIXED_NOREPLACE). * * See below for "brk" handling details, which is * also affected by program vs loader and ASLR. */ if (interpreter) { /* On ET_DYN with PT_INTERP, we do the ASLR. */ load_bias = ELF_ET_DYN_BASE; if (current->flags & PF_RANDOMIZE) load_bias += arch_mmap_rnd(); /* Adjust alignment as requested. */ if (alignment) load_bias &= ~(alignment - 1); elf_flags |= MAP_FIXED_NOREPLACE; } else { /* * For ET_DYN without PT_INTERP, we rely on * the architectures's (potentially ASLR) mmap * base address (via a load_bias of 0). * * When a large alignment is requested, we * must do the allocation at address "0" right * now to discover where things will load so * that we can adjust the resulting alignment. * In this case (load_bias != 0), we can use * MAP_FIXED_NOREPLACE to make sure the mapping * doesn't collide with anything. */ if (alignment > ELF_MIN_ALIGN) { load_bias = elf_load(bprm->file, 0, elf_ppnt, elf_prot, elf_flags, total_size); if (BAD_ADDR(load_bias)) { retval = IS_ERR_VALUE(load_bias) ? PTR_ERR((void*)load_bias) : -EINVAL; goto out_free_dentry; } vm_munmap(load_bias, total_size); /* Adjust alignment as requested. */ if (alignment) load_bias &= ~(alignment - 1); elf_flags |= MAP_FIXED_NOREPLACE; } else load_bias = 0; } /* * Since load_bias is used for all subsequent loading * calculations, we must lower it by the first vaddr * so that the remaining calculations based on the * ELF vaddrs will be correctly offset. The result * is then page aligned. */ load_bias = ELF_PAGESTART(load_bias - vaddr); } error = elf_load(bprm->file, load_bias + vaddr, elf_ppnt, elf_prot, elf_flags, total_size); if (BAD_ADDR(error)) { retval = IS_ERR_VALUE(error) ? PTR_ERR((void*)error) : -EINVAL; goto out_free_dentry; } if (first_pt_load) { first_pt_load = 0; if (elf_ex->e_type == ET_DYN) { load_bias += error - ELF_PAGESTART(load_bias + vaddr); reloc_func_desc = load_bias; } } /* * Figure out which segment in the file contains the Program * Header table, and map to the associated memory address. */ if (elf_ppnt->p_offset <= elf_ex->e_phoff && elf_ex->e_phoff < elf_ppnt->p_offset + elf_ppnt->p_filesz) { phdr_addr = elf_ex->e_phoff - elf_ppnt->p_offset + elf_ppnt->p_vaddr; } k = elf_ppnt->p_vaddr; if ((elf_ppnt->p_flags & PF_X) && k < start_code) start_code = k; if (start_data < k) start_data = k; /* * Check to see if the section's size will overflow the * allowed task size. Note that p_filesz must always be * <= p_memsz so it is only necessary to check p_memsz. */ if (BAD_ADDR(k) || elf_ppnt->p_filesz > elf_ppnt->p_memsz || elf_ppnt->p_memsz > TASK_SIZE || TASK_SIZE - elf_ppnt->p_memsz < k) { /* set_brk can never work. Avoid overflows. */ retval = -EINVAL; goto out_free_dentry; } k = elf_ppnt->p_vaddr + elf_ppnt->p_filesz; if ((elf_ppnt->p_flags & PF_X) && end_code < k) end_code = k; if (end_data < k) end_data = k; k = elf_ppnt->p_vaddr + elf_ppnt->p_memsz; if (k > elf_brk) elf_brk = k; } e_entry = elf_ex->e_entry + load_bias; phdr_addr += load_bias; elf_brk += load_bias; start_code += load_bias; end_code += load_bias; start_data += load_bias; end_data += load_bias; if (interpreter) { elf_entry = load_elf_interp(interp_elf_ex, interpreter, load_bias, interp_elf_phdata, &arch_state); if (!IS_ERR_VALUE(elf_entry)) { /* * load_elf_interp() returns relocation * adjustment */ interp_load_addr = elf_entry; elf_entry += interp_elf_ex->e_entry; } if (BAD_ADDR(elf_entry)) { retval = IS_ERR_VALUE(elf_entry) ? (int)elf_entry : -EINVAL; goto out_free_dentry; } reloc_func_desc = interp_load_addr; exe_file_allow_write_access(interpreter); fput(interpreter); kfree(interp_elf_ex); kfree(interp_elf_phdata); } else { elf_entry = e_entry; if (BAD_ADDR(elf_entry)) { retval = -EINVAL; goto out_free_dentry; } } kfree(elf_phdata); set_binfmt(&elf_format); #ifdef ARCH_HAS_SETUP_ADDITIONAL_PAGES retval = ARCH_SETUP_ADDITIONAL_PAGES(bprm, elf_ex, !!interpreter); if (retval < 0) goto out; #endif /* ARCH_HAS_SETUP_ADDITIONAL_PAGES */ retval = create_elf_tables(bprm, elf_ex, interp_load_addr, e_entry, phdr_addr); if (retval < 0) goto out; mm = current->mm; mm->end_code = end_code; mm->start_code = start_code; mm->start_data = start_data; mm->end_data = end_data; mm->start_stack = bprm->p; elf_coredump_set_mm_eflags(mm, elf_ex->e_flags); /** * DOC: "brk" handling * * For architectures with ELF randomization, when executing a * loader directly (i.e. static PIE: ET_DYN without PT_INTERP), * move the brk area out of the mmap region and into the unused * ELF_ET_DYN_BASE region. Since "brk" grows up it may collide * early with the stack growing down or other regions being put * into the mmap region by the kernel (e.g. vdso). * * In the CONFIG_COMPAT_BRK case, though, everything is turned * off because we're not allowed to move the brk at all. */ if (!IS_ENABLED(CONFIG_COMPAT_BRK) && IS_ENABLED(CONFIG_ARCH_HAS_ELF_RANDOMIZE) && elf_ex->e_type == ET_DYN && !interpreter) { elf_brk = ELF_ET_DYN_BASE; /* This counts as moving the brk, so let brk(2) know. */ brk_moved = true; } mm->start_brk = mm->brk = ELF_PAGEALIGN(elf_brk); if ((current->flags & PF_RANDOMIZE) && snapshot_randomize_va_space > 1) { /* * If we didn't move the brk to ELF_ET_DYN_BASE (above), * leave a gap between .bss and brk. */ if (!brk_moved) mm->brk = mm->start_brk = mm->brk + PAGE_SIZE; mm->brk = mm->start_brk = arch_randomize_brk(mm); brk_moved = true; } #ifdef compat_brk_randomized if (brk_moved) current->brk_randomized = 1; #endif if (current->personality & MMAP_PAGE_ZERO) { /* Why this, you ask??? Well SVr4 maps page 0 as read-only, and some applications "depend" upon this behavior. Since we do not have the power to recompile these, we emulate the SVr4 behavior. Sigh. */ error = vm_mmap(NULL, 0, PAGE_SIZE, PROT_READ | PROT_EXEC, MAP_FIXED | MAP_PRIVATE, 0); retval = do_mseal(0, PAGE_SIZE, 0); if (retval) pr_warn_ratelimited("pid=%d, couldn't seal address 0, ret=%d.\n", task_pid_nr(current), retval); } regs = current_pt_regs(); #ifdef ELF_PLAT_INIT /* * The ABI may specify that certain registers be set up in special * ways (on i386 %edx is the address of a DT_FINI function, for * example. In addition, it may also specify (eg, PowerPC64 ELF) * that the e_entry field is the address of the function descriptor * for the startup routine, rather than the address of the startup * routine itself. This macro performs whatever initialization to * the regs structure is required as well as any relocations to the * function descriptor entries when executing dynamically links apps. */ ELF_PLAT_INIT(regs, reloc_func_desc); #endif finalize_exec(bprm); START_THREAD(elf_ex, regs, elf_entry, bprm->p); retval = 0; out: return retval; /* error cleanup */ out_free_dentry: kfree(interp_elf_ex); kfree(interp_elf_phdata); out_free_file: exe_file_allow_write_access(interpreter); if (interpreter) fput(interpreter); out_free_ph: kfree(elf_phdata); goto out; } #ifdef CONFIG_ELF_CORE /* * ELF core dumper * * Modelled on fs/exec.c:aout_core_dump() * Jeremy Fitzhardinge <jeremy@sw.oz.au> */ /* An ELF note in memory */ struct memelfnote { const char *name; int type; unsigned int datasz; void *data; }; static int notesize(struct memelfnote *en) { int sz; sz = sizeof(struct elf_note); sz += roundup(strlen(en->name) + 1, 4); sz += roundup(en->datasz, 4); return sz; } static int writenote(struct memelfnote *men, struct coredump_params *cprm) { struct elf_note en; en.n_namesz = strlen(men->name) + 1; en.n_descsz = men->datasz; en.n_type = men->type; return dump_emit(cprm, &en, sizeof(en)) && dump_emit(cprm, men->name, en.n_namesz) && dump_align(cprm, 4) && dump_emit(cprm, men->data, men->datasz) && dump_align(cprm, 4); } static void fill_elf_header(struct elfhdr *elf, int segs, u16 machine, u32 flags) { memset(elf, 0, sizeof(*elf)); memcpy(elf->e_ident, ELFMAG, SELFMAG); elf->e_ident[EI_CLASS] = ELF_CLASS; elf->e_ident[EI_DATA] = ELF_DATA; elf->e_ident[EI_VERSION] = EV_CURRENT; elf->e_ident[EI_OSABI] = ELF_OSABI; elf->e_type = ET_CORE; elf->e_machine = machine; elf->e_version = EV_CURRENT; elf->e_phoff = sizeof(struct elfhdr); elf->e_flags = flags; elf->e_ehsize = sizeof(struct elfhdr); elf->e_phentsize = sizeof(struct elf_phdr); elf->e_phnum = segs; } static void fill_elf_note_phdr(struct elf_phdr *phdr, int sz, loff_t offset) { phdr->p_type = PT_NOTE; phdr->p_offset = offset; phdr->p_vaddr = 0; phdr->p_paddr = 0; phdr->p_filesz = sz; phdr->p_memsz = 0; phdr->p_flags = 0; phdr->p_align = 4; } static void __fill_note(struct memelfnote *note, const char *name, int type, unsigned int sz, void *data) { note->name = name; note->type = type; note->datasz = sz; note->data = data; } #define fill_note(note, type, sz, data) \ __fill_note(note, NN_ ## type, NT_ ## type, sz, data) /* * fill up all the fields in prstatus from the given task struct, except * registers which need to be filled up separately. */ static void fill_prstatus(struct elf_prstatus_common *prstatus, struct task_struct *p, long signr) { prstatus->pr_info.si_signo = prstatus->pr_cursig = signr; prstatus->pr_sigpend = p->pending.signal.sig[0]; prstatus->pr_sighold = p->blocked.sig[0]; rcu_read_lock(); prstatus->pr_ppid = task_pid_vnr(rcu_dereference(p->real_parent)); rcu_read_unlock(); prstatus->pr_pid = task_pid_vnr(p); prstatus->pr_pgrp = task_pgrp_vnr(p); prstatus->pr_sid = task_session_vnr(p); if (thread_group_leader(p)) { struct task_cputime cputime; /* * This is the record for the group leader. It shows the * group-wide total, not its individual thread total. */ thread_group_cputime(p, &cputime); prstatus->pr_utime = ns_to_kernel_old_timeval(cputime.utime); prstatus->pr_stime = ns_to_kernel_old_timeval(cputime.stime); } else { u64 utime, stime; task_cputime(p, &utime, &stime); prstatus->pr_utime = ns_to_kernel_old_timeval(utime); prstatus->pr_stime = ns_to_kernel_old_timeval(stime); } prstatus->pr_cutime = ns_to_kernel_old_timeval(p->signal->cutime); prstatus->pr_cstime = ns_to_kernel_old_timeval(p->signal->cstime); } static int fill_psinfo(struct elf_prpsinfo *psinfo, struct task_struct *p, struct mm_struct *mm) { const struct cred *cred; unsigned int i, len; unsigned int state; /* first copy the parameters from user space */ memset(psinfo, 0, sizeof(struct elf_prpsinfo)); len = mm->arg_end - mm->arg_start; if (len >= ELF_PRARGSZ) len = ELF_PRARGSZ-1; if (copy_from_user(&psinfo->pr_psargs, (const char __user *)mm->arg_start, len)) return -EFAULT; for(i = 0; i < len; i++) if (psinfo->pr_psargs[i] == 0) psinfo->pr_psargs[i] = ' '; psinfo->pr_psargs[len] = 0; rcu_read_lock(); psinfo->pr_ppid = task_pid_vnr(rcu_dereference(p->real_parent)); rcu_read_unlock(); psinfo->pr_pid = task_pid_vnr(p); psinfo->pr_pgrp = task_pgrp_vnr(p); psinfo->pr_sid = task_session_vnr(p); state = READ_ONCE(p->__state); i = state ? ffz(~state) + 1 : 0; psinfo->pr_state = i; psinfo->pr_sname = (i > 5) ? '.' : "RSDTZW"[i]; psinfo->pr_zomb = psinfo->pr_sname == 'Z'; psinfo->pr_nice = task_nice(p); psinfo->pr_flag = p->flags; rcu_read_lock(); cred = __task_cred(p); SET_UID(psinfo->pr_uid, from_kuid_munged(cred->user_ns, cred->uid)); SET_GID(psinfo->pr_gid, from_kgid_munged(cred->user_ns, cred->gid)); rcu_read_unlock(); get_task_comm(psinfo->pr_fname, p); return 0; } static void fill_auxv_note(struct memelfnote *note, struct mm_struct *mm) { elf_addr_t *auxv = (elf_addr_t *) mm->saved_auxv; int i = 0; do i += 2; while (auxv[i - 2] != AT_NULL); fill_note(note, AUXV, i * sizeof(elf_addr_t), auxv); } static void fill_siginfo_note(struct memelfnote *note, user_siginfo_t *csigdata, const kernel_siginfo_t *siginfo) { copy_siginfo_to_external(csigdata, siginfo); fill_note(note, SIGINFO, sizeof(*csigdata), csigdata); } /* * Format of NT_FILE note: * * long count -- how many files are mapped * long page_size -- units for file_ofs * array of [COUNT] elements of * long start * long end * long file_ofs * followed by COUNT filenames in ASCII: "FILE1" NUL "FILE2" NUL... */ static int fill_files_note(struct memelfnote *note, struct coredump_params *cprm) { unsigned count, size, names_ofs, remaining, n; user_long_t *data; user_long_t *start_end_ofs; char *name_base, *name_curpos; int i; /* *Estimated* file count and total data size needed */ count = cprm->vma_count; if (count > UINT_MAX / 64) return -EINVAL; size = count * 64; names_ofs = (2 + 3 * count) * sizeof(data[0]); alloc: /* paranoia check */ if (size >= core_file_note_size_limit) { pr_warn_once("coredump Note size too large: %u (does kernel.core_file_note_size_limit sysctl need adjustment?\n", size); return -EINVAL; } size = round_up(size, PAGE_SIZE); /* * "size" can be 0 here legitimately. * Let it ENOMEM and omit NT_FILE section which will be empty anyway. */ data = kvmalloc(size, GFP_KERNEL); if (ZERO_OR_NULL_PTR(data)) return -ENOMEM; start_end_ofs = data + 2; name_base = name_curpos = ((char *)data) + names_ofs; remaining = size - names_ofs; count = 0; for (i = 0; i < cprm->vma_count; i++) { struct core_vma_metadata *m = &cprm->vma_meta[i]; struct file *file; const char *filename; file = m->file; if (!file) continue; filename = file_path(file, name_curpos, remaining); if (IS_ERR(filename)) { if (PTR_ERR(filename) == -ENAMETOOLONG) { kvfree(data); size = size * 5 / 4; goto alloc; } continue; } /* file_path() fills at the end, move name down */ /* n = strlen(filename) + 1: */ n = (name_curpos + remaining) - filename; remaining = filename - name_curpos; memmove(name_curpos, filename, n); name_curpos += n; *start_end_ofs++ = m->start; *start_end_ofs++ = m->end; *start_end_ofs++ = m->pgoff; count++; } /* Now we know exact count of files, can store it */ data[0] = count; data[1] = PAGE_SIZE; /* * Count usually is less than mm->map_count, * we need to move filenames down. */ n = cprm->vma_count - count; if (n != 0) { unsigned shift_bytes = n * 3 * sizeof(data[0]); memmove(name_base - shift_bytes, name_base, name_curpos - name_base); name_curpos -= shift_bytes; } size = name_curpos - (char *)data; fill_note(note, FILE, size, data); return 0; } #include <linux/regset.h> struct elf_thread_core_info { struct elf_thread_core_info *next; struct task_struct *task; struct elf_prstatus prstatus; struct memelfnote notes[]; }; struct elf_note_info { struct elf_thread_core_info *thread; struct memelfnote psinfo; struct memelfnote signote; struct memelfnote auxv; struct memelfnote files; user_siginfo_t csigdata; size_t size; int thread_notes; }; #ifdef CORE_DUMP_USE_REGSET /* * When a regset has a writeback hook, we call it on each thread before * dumping user memory. On register window machines, this makes sure the * user memory backing the register data is up to date before we read it. */ static void do_thread_regset_writeback(struct task_struct *task, const struct user_regset *regset) { if (regset->writeback) regset->writeback(task, regset, 1); } #ifndef PRSTATUS_SIZE #define PRSTATUS_SIZE sizeof(struct elf_prstatus) #endif #ifndef SET_PR_FPVALID #define SET_PR_FPVALID(S) ((S)->pr_fpvalid = 1) #endif static int fill_thread_core_info(struct elf_thread_core_info *t, const struct user_regset_view *view, long signr, struct elf_note_info *info) { unsigned int note_iter, view_iter; /* * NT_PRSTATUS is the one special case, because the regset data * goes into the pr_reg field inside the note contents, rather * than being the whole note contents. We fill the regset in here. * We assume that regset 0 is NT_PRSTATUS. */ fill_prstatus(&t->prstatus.common, t->task, signr); regset_get(t->task, &view->regsets[0], sizeof(t->prstatus.pr_reg), &t->prstatus.pr_reg); fill_note(&t->notes[0], PRSTATUS, PRSTATUS_SIZE, &t->prstatus); info->size += notesize(&t->notes[0]); do_thread_regset_writeback(t->task, &view->regsets[0]); /* * Each other regset might generate a note too. For each regset * that has no core_note_type or is inactive, skip it. */ note_iter = 1; for (view_iter = 1; view_iter < view->n; ++view_iter) { const struct user_regset *regset = &view->regsets[view_iter]; int note_type = regset->core_note_type; const char *note_name = regset->core_note_name; bool is_fpreg = note_type == NT_PRFPREG; void *data; int ret; do_thread_regset_writeback(t->task, regset); if (!note_type) // not for coredumps continue; if (regset->active && regset->active(t->task, regset) <= 0) continue; ret = regset_get_alloc(t->task, regset, ~0U, &data); if (ret < 0) continue; if (WARN_ON_ONCE(note_iter >= info->thread_notes)) break; if (is_fpreg) SET_PR_FPVALID(&t->prstatus); /* There should be a note name, but if not, guess: */ if (WARN_ON_ONCE(!note_name)) note_name = "LINUX"; else /* Warn on non-legacy-compatible names, for now. */ WARN_ON_ONCE(strcmp(note_name, is_fpreg ? "CORE" : "LINUX")); __fill_note(&t->notes[note_iter], note_name, note_type, ret, data); info->size += notesize(&t->notes[note_iter]); note_iter++; } return 1; } #else static int fill_thread_core_info(struct elf_thread_core_info *t, const struct user_regset_view *view, long signr, struct elf_note_info *info) { struct task_struct *p = t->task; elf_fpregset_t *fpu; fill_prstatus(&t->prstatus.common, p, signr); elf_core_copy_task_regs(p, &t->prstatus.pr_reg); fill_note(&t->notes[0], PRSTATUS, sizeof(t->prstatus), &t->prstatus); info->size += notesize(&t->notes[0]); fpu = kzalloc(sizeof(elf_fpregset_t), GFP_KERNEL); if (!fpu || !elf_core_copy_task_fpregs(p, fpu)) { kfree(fpu); return 1; } t->prstatus.pr_fpvalid = 1; fill_note(&t->notes[1], PRFPREG, sizeof(*fpu), fpu); info->size += notesize(&t->notes[1]); return 1; } #endif static int fill_note_info(struct elfhdr *elf, int phdrs, struct elf_note_info *info, struct coredump_params *cprm) { struct task_struct *dump_task = current; const struct user_regset_view *view; struct elf_thread_core_info *t; struct elf_prpsinfo *psinfo; struct core_thread *ct; u16 machine; u32 flags; psinfo = kmalloc(sizeof(*psinfo), GFP_KERNEL); if (!psinfo) return 0; fill_note(&info->psinfo, PRPSINFO, sizeof(*psinfo), psinfo); #ifdef CORE_DUMP_USE_REGSET view = task_user_regset_view(dump_task); /* * Figure out how many notes we're going to need for each thread. */ info->thread_notes = 0; for (int i = 0; i < view->n; ++i) if (view->regsets[i].core_note_type != 0) ++info->thread_notes; /* * Sanity check. |