| 1 1 1 1 1 1 1 1 1 8 7 8 7 7 6 1 5 5 2 2 2 1 4 4 1 2 3 8 3 3 3 3 2 1 2 48 48 2 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 | // SPDX-License-Identifier: GPL-2.0-or-later /* * net/sched/act_connmark.c netfilter connmark retriever action * skb mark is over-written * * Copyright (c) 2011 Felix Fietkau <nbd@openwrt.org> */ #include <linux/module.h> #include <linux/init.h> #include <linux/kernel.h> #include <linux/skbuff.h> #include <linux/rtnetlink.h> #include <linux/pkt_cls.h> #include <linux/ip.h> #include <linux/ipv6.h> #include <net/netlink.h> #include <net/pkt_sched.h> #include <net/act_api.h> #include <net/pkt_cls.h> #include <uapi/linux/tc_act/tc_connmark.h> #include <net/tc_act/tc_connmark.h> #include <net/tc_wrapper.h> #include <net/netfilter/nf_conntrack.h> #include <net/netfilter/nf_conntrack_core.h> #include <net/netfilter/nf_conntrack_zones.h> static struct tc_action_ops act_connmark_ops; TC_INDIRECT_SCOPE int tcf_connmark_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) { const struct nf_conntrack_tuple_hash *thash; struct nf_conntrack_tuple tuple; enum ip_conntrack_info ctinfo; struct tcf_connmark_info *ca = to_connmark(a); struct tcf_connmark_parms *parms; struct nf_conntrack_zone zone; struct nf_conn *c; int proto; tcf_lastuse_update(&ca->tcf_tm); tcf_action_update_bstats(&ca->common, skb); parms = rcu_dereference_bh(ca->parms); switch (skb_protocol(skb, true)) { case htons(ETH_P_IP): if (skb->len < sizeof(struct iphdr)) goto out; proto = NFPROTO_IPV4; break; case htons(ETH_P_IPV6): if (skb->len < sizeof(struct ipv6hdr)) goto out; proto = NFPROTO_IPV6; break; default: goto out; } c = nf_ct_get(skb, &ctinfo); if (c) { skb->mark = READ_ONCE(c->mark); goto count; } if (!nf_ct_get_tuplepr(skb, skb_network_offset(skb), proto, parms->net, &tuple)) goto out; zone.id = parms->zone; zone.dir = NF_CT_DEFAULT_ZONE_DIR; thash = nf_conntrack_find_get(parms->net, &zone, &tuple); if (!thash) goto out; c = nf_ct_tuplehash_to_ctrack(thash); skb->mark = READ_ONCE(c->mark); nf_ct_put(c); count: /* using overlimits stats to count how many packets marked */ tcf_action_inc_overlimit_qstats(&ca->common); out: return parms->action; } static const struct nla_policy connmark_policy[TCA_CONNMARK_MAX + 1] = { [TCA_CONNMARK_PARMS] = { .len = sizeof(struct tc_connmark) }, }; static int tcf_connmark_init(struct net *net, struct nlattr *nla, struct nlattr *est, struct tc_action **a, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { struct tc_action_net *tn = net_generic(net, act_connmark_ops.net_id); struct tcf_connmark_parms *nparms, *oparms; struct nlattr *tb[TCA_CONNMARK_MAX + 1]; bool bind = flags & TCA_ACT_FLAGS_BIND; struct tcf_chain *goto_ch = NULL; struct tcf_connmark_info *ci; struct tc_connmark *parm; int ret = 0, err; u32 index; if (!nla) return -EINVAL; ret = nla_parse_nested_deprecated(tb, TCA_CONNMARK_MAX, nla, connmark_policy, NULL); if (ret < 0) return ret; if (!tb[TCA_CONNMARK_PARMS]) return -EINVAL; nparms = kzalloc_obj(*nparms); if (!nparms) return -ENOMEM; parm = nla_data(tb[TCA_CONNMARK_PARMS]); index = parm->index; ret = tcf_idr_check_alloc(tn, &index, a, bind); if (!ret) { ret = tcf_idr_create_from_flags(tn, index, est, a, &act_connmark_ops, bind, flags); if (ret) { tcf_idr_cleanup(tn, index); err = ret; goto out_free; } ci = to_connmark(*a); nparms->net = net; nparms->zone = parm->zone; ret = ACT_P_CREATED; } else if (ret > 0) { ci = to_connmark(*a); if (bind) { err = ACT_P_BOUND; goto out_free; } if (!(flags & TCA_ACT_FLAGS_REPLACE)) { err = -EEXIST; goto release_idr; } nparms->net = rtnl_dereference(ci->parms)->net; nparms->zone = parm->zone; ret = 0; } else { err = ret; goto out_free; } err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); if (err < 0) goto release_idr; nparms->action = parm->action; spin_lock_bh(&ci->tcf_lock); goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); oparms = rcu_replace_pointer(ci->parms, nparms, lockdep_is_held(&ci->tcf_lock)); spin_unlock_bh(&ci->tcf_lock); if (goto_ch) tcf_chain_put_by_act(goto_ch); if (oparms) kfree_rcu(oparms, rcu); return ret; release_idr: tcf_idr_release(*a, bind); out_free: kfree(nparms); return err; } static inline int tcf_connmark_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { const struct tcf_connmark_info *ci = to_connmark(a); unsigned char *b = skb_tail_pointer(skb); const struct tcf_connmark_parms *parms; struct tc_connmark opt; struct tcf_t t; memset(&opt, 0, sizeof(opt)); opt.index = ci->tcf_index; opt.refcnt = refcount_read(&ci->tcf_refcnt) - ref; opt.bindcnt = atomic_read(&ci->tcf_bindcnt) - bind; rcu_read_lock(); parms = rcu_dereference(ci->parms); opt.action = parms->action; opt.zone = parms->zone; if (nla_put(skb, TCA_CONNMARK_PARMS, sizeof(opt), &opt)) goto nla_put_failure; tcf_tm_dump(&t, &ci->tcf_tm); if (nla_put_64bit(skb, TCA_CONNMARK_TM, sizeof(t), &t, TCA_CONNMARK_PAD)) goto nla_put_failure; rcu_read_unlock(); return skb->len; nla_put_failure: rcu_read_unlock(); nlmsg_trim(skb, b); return -1; } static void tcf_connmark_cleanup(struct tc_action *a) { struct tcf_connmark_info *ci = to_connmark(a); struct tcf_connmark_parms *parms; parms = rcu_dereference_protected(ci->parms, 1); if (parms) kfree_rcu(parms, rcu); } static struct tc_action_ops act_connmark_ops = { .kind = "connmark", .id = TCA_ID_CONNMARK, .owner = THIS_MODULE, .act = tcf_connmark_act, .dump = tcf_connmark_dump, .init = tcf_connmark_init, .cleanup = tcf_connmark_cleanup, .size = sizeof(struct tcf_connmark_info), }; MODULE_ALIAS_NET_ACT("connmark"); static __net_init int connmark_init_net(struct net *net) { struct tc_action_net *tn = net_generic(net, act_connmark_ops.net_id); return tc_action_net_init(net, tn, &act_connmark_ops); } static void __net_exit connmark_exit_net(struct list_head *net_list) { tc_action_net_exit(net_list, act_connmark_ops.net_id); } static struct pernet_operations connmark_net_ops = { .init = connmark_init_net, .exit_batch = connmark_exit_net, .id = &act_connmark_ops.net_id, .size = sizeof(struct tc_action_net), }; static int __init connmark_init_module(void) { return tcf_register_action(&act_connmark_ops, &connmark_net_ops); } static void __exit connmark_cleanup_module(void) { tcf_unregister_action(&act_connmark_ops, &connmark_net_ops); } module_init(connmark_init_module); module_exit(connmark_cleanup_module); MODULE_AUTHOR("Felix Fietkau <nbd@openwrt.org>"); MODULE_DESCRIPTION("Connection tracking mark restoring"); MODULE_LICENSE("GPL"); |
| 4 571 466 1085 740 1241 123 299 335 323 25 8 855 6 510 524 24 2 405 99 406 181 179 407 804 800 804 801 804 795 806 221 1056 1028 806 706 17 11 20 5 387 393 12 3 3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _NET_NEIGHBOUR_H #define _NET_NEIGHBOUR_H #include <linux/neighbour.h> /* * Generic neighbour manipulation * * Authors: * Pedro Roque <roque@di.fc.ul.pt> * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> * * Changes: * * Harald Welte: <laforge@gnumonks.org> * - Add neighbour cache statistics like rtstat */ #include <linux/atomic.h> #include <linux/refcount.h> #include <linux/netdevice.h> #include <linux/skbuff.h> #include <linux/rcupdate.h> #include <linux/seq_file.h> #include <linux/bitmap.h> #include <linux/err.h> #include <linux/sysctl.h> #include <linux/workqueue.h> #include <net/rtnetlink.h> #include <net/neighbour_tables.h> /* * NUD stands for "neighbor unreachability detection" */ #define NUD_IN_TIMER (NUD_INCOMPLETE|NUD_REACHABLE|NUD_DELAY|NUD_PROBE) #define NUD_VALID (NUD_PERMANENT|NUD_NOARP|NUD_REACHABLE|NUD_PROBE|NUD_STALE|NUD_DELAY) #define NUD_CONNECTED (NUD_PERMANENT|NUD_NOARP|NUD_REACHABLE) struct neighbour; enum { NEIGH_VAR_MCAST_PROBES, NEIGH_VAR_UCAST_PROBES, NEIGH_VAR_APP_PROBES, NEIGH_VAR_MCAST_REPROBES, NEIGH_VAR_RETRANS_TIME, NEIGH_VAR_BASE_REACHABLE_TIME, NEIGH_VAR_DELAY_PROBE_TIME, NEIGH_VAR_INTERVAL_PROBE_TIME_MS, NEIGH_VAR_GC_STALETIME, NEIGH_VAR_QUEUE_LEN_BYTES, NEIGH_VAR_PROXY_QLEN, NEIGH_VAR_ANYCAST_DELAY, NEIGH_VAR_PROXY_DELAY, NEIGH_VAR_LOCKTIME, #define NEIGH_VAR_DATA_MAX (NEIGH_VAR_LOCKTIME + 1) /* Following are used as a second way to access one of the above */ NEIGH_VAR_QUEUE_LEN, /* same data as NEIGH_VAR_QUEUE_LEN_BYTES */ NEIGH_VAR_RETRANS_TIME_MS, /* same data as NEIGH_VAR_RETRANS_TIME */ NEIGH_VAR_BASE_REACHABLE_TIME_MS, /* same data as NEIGH_VAR_BASE_REACHABLE_TIME */ /* Following are used by "default" only */ NEIGH_VAR_GC_INTERVAL, NEIGH_VAR_GC_THRESH1, NEIGH_VAR_GC_THRESH2, NEIGH_VAR_GC_THRESH3, NEIGH_VAR_MAX }; struct neigh_parms { possible_net_t net; struct net_device *dev; netdevice_tracker dev_tracker; struct list_head list; int (*neigh_setup)(struct neighbour *); struct neigh_table *tbl; void *sysctl_table; int dead; refcount_t refcnt; struct rcu_head rcu_head; int reachable_time; u32 qlen; int data[NEIGH_VAR_DATA_MAX]; DECLARE_BITMAP(data_state, NEIGH_VAR_DATA_MAX); }; static inline void neigh_var_set(struct neigh_parms *p, int index, int val) { set_bit(index, p->data_state); WRITE_ONCE(p->data[index], val); } #define __NEIGH_VAR(p, attr) ((p)->data[NEIGH_VAR_ ## attr]) #define NEIGH_VAR(p, attr) READ_ONCE(__NEIGH_VAR(p, attr)) #define NEIGH_VAR_PTR(p, attr) (&(__NEIGH_VAR(p, attr))) /* In ndo_neigh_setup, NEIGH_VAR_INIT should be used. * In other cases, NEIGH_VAR_SET should be used. */ #define NEIGH_VAR_INIT(p, attr, val) (__NEIGH_VAR(p, attr) = val) #define NEIGH_VAR_SET(p, attr, val) neigh_var_set(p, NEIGH_VAR_ ## attr, val) static inline void neigh_parms_data_state_setall(struct neigh_parms *p) { bitmap_fill(p->data_state, NEIGH_VAR_DATA_MAX); } static inline void neigh_parms_data_state_cleanall(struct neigh_parms *p) { bitmap_zero(p->data_state, NEIGH_VAR_DATA_MAX); } struct neigh_statistics { unsigned long allocs; /* number of allocated neighs */ unsigned long destroys; /* number of destroyed neighs */ unsigned long hash_grows; /* number of hash resizes */ unsigned long res_failed; /* number of failed resolutions */ unsigned long lookups; /* number of lookups */ unsigned long hits; /* number of hits (among lookups) */ unsigned long rcv_probes_mcast; /* number of received mcast ipv6 */ unsigned long rcv_probes_ucast; /* number of received ucast ipv6 */ unsigned long periodic_gc_runs; /* number of periodic GC runs */ unsigned long forced_gc_runs; /* number of forced GC runs */ unsigned long unres_discards; /* number of unresolved drops */ unsigned long table_fulls; /* times even gc couldn't help */ }; #define NEIGH_CACHE_STAT_INC(tbl, field) this_cpu_inc((tbl)->stats->field) struct neighbour { struct hlist_node hash; struct hlist_node dev_list; struct neigh_table *tbl; struct neigh_parms *parms; unsigned long confirmed; unsigned long updated; rwlock_t lock; refcount_t refcnt; unsigned int arp_queue_len_bytes; struct sk_buff_head arp_queue; struct timer_list timer; unsigned long used; atomic_t probes; u8 nud_state; u8 type; u8 dead; u8 protocol; u32 flags; seqlock_t ha_lock; unsigned char ha[ALIGN(MAX_ADDR_LEN, sizeof(unsigned long))] __aligned(8); struct hh_cache hh; int (*output)(struct neighbour *, struct sk_buff *); const struct neigh_ops *ops; struct list_head gc_list; struct list_head managed_list; struct rcu_head rcu; struct net_device *dev; netdevice_tracker dev_tracker; u8 primary_key[]; } __randomize_layout; struct neigh_ops { int family; void (*solicit)(struct neighbour *, struct sk_buff *); void (*error_report)(struct neighbour *, struct sk_buff *); int (*output)(struct neighbour *, struct sk_buff *); int (*connected_output)(struct neighbour *, struct sk_buff *); }; struct pneigh_entry { struct pneigh_entry __rcu *next; possible_net_t net; struct net_device *dev; netdevice_tracker dev_tracker; union { struct list_head free_node; struct rcu_head rcu; }; u32 flags; u8 protocol; bool permanent; u32 key[]; }; /* * neighbour table manipulation */ #define NEIGH_NUM_HASH_RND 4 struct neigh_hash_table { struct hlist_head *hash_heads; unsigned int hash_shift; __u32 hash_rnd[NEIGH_NUM_HASH_RND]; struct rcu_head rcu; }; struct neigh_table { int family; unsigned int entry_size; unsigned int key_len; __be16 protocol; __u32 (*hash)(const void *pkey, const struct net_device *dev, __u32 *hash_rnd); bool (*key_eq)(const struct neighbour *, const void *pkey); int (*constructor)(struct neighbour *); int (*pconstructor)(struct pneigh_entry *); void (*pdestructor)(struct pneigh_entry *); void (*proxy_redo)(struct sk_buff *skb); int (*is_multicast)(const void *pkey); bool (*allow_add)(const struct net_device *dev, struct netlink_ext_ack *extack); char *id; struct neigh_parms parms; struct list_head parms_list; int gc_interval; int gc_thresh1; int gc_thresh2; int gc_thresh3; unsigned long last_flush; struct delayed_work gc_work; struct delayed_work managed_work; struct timer_list proxy_timer; struct sk_buff_head proxy_queue; atomic_t entries; atomic_t gc_entries; struct list_head gc_list; struct list_head managed_list; spinlock_t lock; unsigned long last_rand; struct neigh_statistics __percpu *stats; struct neigh_hash_table __rcu *nht; struct mutex phash_lock; struct pneigh_entry __rcu **phash_buckets; }; static inline int neigh_parms_family(struct neigh_parms *p) { return p->tbl->family; } #define NEIGH_PRIV_ALIGN sizeof(long long) #define NEIGH_ENTRY_SIZE(size) ALIGN((size), NEIGH_PRIV_ALIGN) static inline void *neighbour_priv(const struct neighbour *n) { return (char *)n + n->tbl->entry_size; } /* flags for neigh_update() */ #define NEIGH_UPDATE_F_OVERRIDE BIT(0) #define NEIGH_UPDATE_F_WEAK_OVERRIDE BIT(1) #define NEIGH_UPDATE_F_OVERRIDE_ISROUTER BIT(2) #define NEIGH_UPDATE_F_USE BIT(3) #define NEIGH_UPDATE_F_MANAGED BIT(4) #define NEIGH_UPDATE_F_EXT_LEARNED BIT(5) #define NEIGH_UPDATE_F_ISROUTER BIT(6) #define NEIGH_UPDATE_F_ADMIN BIT(7) #define NEIGH_UPDATE_F_EXT_VALIDATED BIT(8) /* In-kernel representation for NDA_FLAGS_EXT flags: */ #define NTF_OLD_MASK 0xff #define NTF_EXT_SHIFT 8 #define NTF_EXT_MASK (NTF_EXT_MANAGED | NTF_EXT_EXT_VALIDATED) #define NTF_MANAGED (NTF_EXT_MANAGED << NTF_EXT_SHIFT) #define NTF_EXT_VALIDATED (NTF_EXT_EXT_VALIDATED << NTF_EXT_SHIFT) extern const struct nla_policy nda_policy[]; #define neigh_for_each_in_bucket(pos, head) hlist_for_each_entry(pos, head, hash) #define neigh_for_each_in_bucket_rcu(pos, head) \ hlist_for_each_entry_rcu(pos, head, hash) #define neigh_for_each_in_bucket_safe(pos, tmp, head) \ hlist_for_each_entry_safe(pos, tmp, head, hash) static inline bool neigh_key_eq32(const struct neighbour *n, const void *pkey) { return *(const u32 *)n->primary_key == *(const u32 *)pkey; } static inline bool neigh_key_eq128(const struct neighbour *n, const void *pkey) { const u32 *n32 = (const u32 *)n->primary_key; const u32 *p32 = pkey; return ((n32[0] ^ p32[0]) | (n32[1] ^ p32[1]) | (n32[2] ^ p32[2]) | (n32[3] ^ p32[3])) == 0; } static inline struct neighbour *___neigh_lookup_noref( struct neigh_table *tbl, bool (*key_eq)(const struct neighbour *n, const void *pkey), __u32 (*hash)(const void *pkey, const struct net_device *dev, __u32 *hash_rnd), const void *pkey, struct net_device *dev) { struct neigh_hash_table *nht = rcu_dereference(tbl->nht); struct neighbour *n; u32 hash_val; hash_val = hash(pkey, dev, nht->hash_rnd) >> (32 - nht->hash_shift); neigh_for_each_in_bucket_rcu(n, &nht->hash_heads[hash_val]) if (n->dev == dev && key_eq(n, pkey)) return n; return NULL; } static inline struct neighbour *__neigh_lookup_noref(struct neigh_table *tbl, const void *pkey, struct net_device *dev) { return ___neigh_lookup_noref(tbl, tbl->key_eq, tbl->hash, pkey, dev); } static inline void neigh_confirm(struct neighbour *n) { if (n) { unsigned long now = jiffies; /* avoid dirtying neighbour */ if (READ_ONCE(n->confirmed) != now) WRITE_ONCE(n->confirmed, now); } } void neigh_table_init(int index, struct neigh_table *tbl); int neigh_table_clear(int index, struct neigh_table *tbl); struct neighbour *neigh_lookup(struct neigh_table *tbl, const void *pkey, struct net_device *dev); struct neighbour *__neigh_create(struct neigh_table *tbl, const void *pkey, struct net_device *dev, bool want_ref); static inline struct neighbour *neigh_create(struct neigh_table *tbl, const void *pkey, struct net_device *dev) { return __neigh_create(tbl, pkey, dev, true); } void neigh_destroy(struct neighbour *neigh); int __neigh_event_send(struct neighbour *neigh, struct sk_buff *skb, const bool immediate_ok); int neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new, u32 flags, u32 nlmsg_pid); void __neigh_set_probe_once(struct neighbour *neigh); bool neigh_remove_one(struct neighbour *ndel); void neigh_changeaddr(struct neigh_table *tbl, struct net_device *dev); int neigh_ifdown(struct neigh_table *tbl, struct net_device *dev); int neigh_carrier_down(struct neigh_table *tbl, struct net_device *dev); int neigh_resolve_output(struct neighbour *neigh, struct sk_buff *skb); int neigh_connected_output(struct neighbour *neigh, struct sk_buff *skb); int neigh_direct_output(struct neighbour *neigh, struct sk_buff *skb); struct neighbour *neigh_event_ns(struct neigh_table *tbl, u8 *lladdr, void *saddr, struct net_device *dev); struct neigh_parms *neigh_parms_alloc(struct net_device *dev, struct neigh_table *tbl); void neigh_parms_release(struct neigh_table *tbl, struct neigh_parms *parms); static inline struct net *neigh_parms_net(const struct neigh_parms *parms) { return read_pnet(&parms->net); } unsigned long neigh_rand_reach_time(unsigned long base); static inline void neigh_set_reach_time(struct neigh_parms *p) { unsigned long base = NEIGH_VAR(p, BASE_REACHABLE_TIME); WRITE_ONCE(p->reachable_time, neigh_rand_reach_time(base)); } void pneigh_enqueue(struct neigh_table *tbl, struct neigh_parms *p, struct sk_buff *skb); struct pneigh_entry *pneigh_lookup(struct neigh_table *tbl, struct net *net, const void *key, struct net_device *dev); int pneigh_create(struct neigh_table *tbl, struct net *net, const void *key, struct net_device *dev, u32 flags, u8 protocol, bool permanent); int pneigh_delete(struct neigh_table *tbl, struct net *net, const void *key, struct net_device *dev); static inline struct net *pneigh_net(const struct pneigh_entry *pneigh) { return read_pnet(&pneigh->net); } void neigh_app_ns(struct neighbour *n); void neigh_for_each(struct neigh_table *tbl, void (*cb)(struct neighbour *, void *), void *cookie); void __neigh_for_each_release(struct neigh_table *tbl, int (*cb)(struct neighbour *)); int neigh_xmit(int fam, struct net_device *, const void *, struct sk_buff *); struct neigh_seq_state { struct seq_net_private p; struct neigh_table *tbl; struct neigh_hash_table *nht; void *(*neigh_sub_iter)(struct neigh_seq_state *state, struct neighbour *n, loff_t *pos); unsigned int bucket; unsigned int flags; #define NEIGH_SEQ_NEIGH_ONLY 0x00000001 #define NEIGH_SEQ_IS_PNEIGH 0x00000002 #define NEIGH_SEQ_SKIP_NOARP 0x00000004 }; void *neigh_seq_start(struct seq_file *, loff_t *, struct neigh_table *, unsigned int); void *neigh_seq_next(struct seq_file *, void *, loff_t *); void neigh_seq_stop(struct seq_file *, void *); int neigh_proc_dointvec(const struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos); int neigh_proc_dointvec_jiffies(const struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos); int neigh_proc_dointvec_ms_jiffies(const struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos); int neigh_sysctl_register(struct net_device *dev, struct neigh_parms *p, proc_handler *proc_handler); void neigh_sysctl_unregister(struct neigh_parms *p); static inline void __neigh_parms_put(struct neigh_parms *parms) { refcount_dec(&parms->refcnt); } static inline struct neigh_parms *neigh_parms_clone(struct neigh_parms *parms) { refcount_inc(&parms->refcnt); return parms; } /* * Neighbour references */ static inline void neigh_release(struct neighbour *neigh) { if (refcount_dec_and_test(&neigh->refcnt)) neigh_destroy(neigh); } static inline struct neighbour * neigh_clone(struct neighbour *neigh) { if (neigh) refcount_inc(&neigh->refcnt); return neigh; } #define neigh_hold(n) refcount_inc(&(n)->refcnt) static __always_inline int neigh_event_send_probe(struct neighbour *neigh, struct sk_buff *skb, const bool immediate_ok) { unsigned long now = jiffies; if (READ_ONCE(neigh->used) != now) WRITE_ONCE(neigh->used, now); if (!(READ_ONCE(neigh->nud_state) & (NUD_CONNECTED | NUD_DELAY | NUD_PROBE))) return __neigh_event_send(neigh, skb, immediate_ok); return 0; } static inline int neigh_event_send(struct neighbour *neigh, struct sk_buff *skb) { return neigh_event_send_probe(neigh, skb, true); } #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) static inline int neigh_hh_bridge(struct hh_cache *hh, struct sk_buff *skb) { unsigned int seq, hh_alen = HH_DATA_ALIGN(ETH_HLEN); int err; err = skb_cow_head(skb, hh_alen); if (err) return err; do { seq = read_seqbegin(&hh->hh_lock); memcpy(skb->data - hh_alen, hh->hh_data, ETH_ALEN + hh_alen - ETH_HLEN); } while (read_seqretry(&hh->hh_lock, seq)); return 0; } #endif static inline int neigh_hh_output(const struct hh_cache *hh, struct sk_buff *skb) { unsigned int hh_alen = 0; unsigned int seq; unsigned int hh_len; do { seq = read_seqbegin(&hh->hh_lock); hh_len = READ_ONCE(hh->hh_len); if (likely(hh_len <= HH_DATA_MOD)) { hh_alen = HH_DATA_MOD; /* skb_push() would proceed silently if we have room for * the unaligned size but not for the aligned size: * check headroom explicitly. */ if (likely(skb_headroom(skb) >= HH_DATA_MOD)) { /* this is inlined by gcc */ memcpy(skb->data - HH_DATA_MOD, hh->hh_data, HH_DATA_MOD); } } else { hh_alen = HH_DATA_ALIGN(hh_len); if (likely(skb_headroom(skb) >= hh_alen)) { memcpy(skb->data - hh_alen, hh->hh_data, hh_alen); } } } while (read_seqretry(&hh->hh_lock, seq)); if (WARN_ON_ONCE(skb_headroom(skb) < hh_alen)) { kfree_skb(skb); return NET_XMIT_DROP; } __skb_push(skb, hh_len); return dev_queue_xmit(skb); } static inline int neigh_output(struct neighbour *n, struct sk_buff *skb, bool skip_cache) { const struct hh_cache *hh = &n->hh; /* n->nud_state and hh->hh_len could be changed under us. * neigh_hh_output() is taking care of the race later. */ if (!skip_cache && (READ_ONCE(n->nud_state) & NUD_CONNECTED) && READ_ONCE(hh->hh_len)) return neigh_hh_output(hh, skb); return READ_ONCE(n->output)(n, skb); } static inline struct neighbour * __neigh_lookup(struct neigh_table *tbl, const void *pkey, struct net_device *dev, int creat) { struct neighbour *n = neigh_lookup(tbl, pkey, dev); if (n || !creat) return n; n = neigh_create(tbl, pkey, dev); return IS_ERR(n) ? NULL : n; } static inline struct neighbour * __neigh_lookup_errno(struct neigh_table *tbl, const void *pkey, struct net_device *dev) { struct neighbour *n = neigh_lookup(tbl, pkey, dev); if (n) return n; return neigh_create(tbl, pkey, dev); } struct neighbour_cb { unsigned long sched_next; unsigned int flags; }; #define LOCALLY_ENQUEUED 0x1 #define NEIGH_CB(skb) ((struct neighbour_cb *)(skb)->cb) static inline void neigh_ha_snapshot(char *dst, const struct neighbour *n, const struct net_device *dev) { unsigned int seq; do { seq = read_seqbegin(&n->ha_lock); memcpy(dst, n->ha, dev->addr_len); } while (read_seqretry(&n->ha_lock, seq)); } static inline void neigh_update_is_router(struct neighbour *neigh, u32 flags, int *notify) { u8 ndm_flags = 0; ndm_flags |= (flags & NEIGH_UPDATE_F_ISROUTER) ? NTF_ROUTER : 0; if ((neigh->flags ^ ndm_flags) & NTF_ROUTER) { if (ndm_flags & NTF_ROUTER) neigh->flags |= NTF_ROUTER; else neigh->flags &= ~NTF_ROUTER; *notify = 1; } } #endif |
| 10 2 2 2 8 8 3 3 3 3 11 11 2 2 2 2 2 1 2 2 2 2 2 2 2 2 2 1 2 2 2 2 2 2 2 2 2 2 6 1 6 5 5 5 5 5 5 5 5 5 5 5 8 8 8 8 6 2 8 8 8 8 8 8 8 8 8 8 1 1 5 5 8 7 8 2 2 2 2 1 10 11 11 2 5 5 5 5 5 5 5 11 5 5 8 8 8 8 8 2 2 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 | // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2001, 2002 Sistina Software (UK) Limited. * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved. * * This file is released under the GPL. */ #include "dm-core.h" #include "dm-rq.h" #include "dm-uevent.h" #include "dm-ima.h" #include <linux/bio-integrity.h> #include <linux/init.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/sched/mm.h> #include <linux/sched/signal.h> #include <linux/blkpg.h> #include <linux/bio.h> #include <linux/mempool.h> #include <linux/dax.h> #include <linux/slab.h> #include <linux/idr.h> #include <linux/uio.h> #include <linux/hdreg.h> #include <linux/delay.h> #include <linux/wait.h> #include <linux/pr.h> #include <linux/refcount.h> #include <linux/part_stat.h> #include <linux/blk-crypto.h> #include <linux/blk-crypto-profile.h> #define DM_MSG_PREFIX "core" /* * Cookies are numeric values sent with CHANGE and REMOVE * uevents while resuming, removing or renaming the device. */ #define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE" #define DM_COOKIE_LENGTH 24 /* * For REQ_POLLED fs bio, this flag is set if we link mapped underlying * dm_io into one list, and reuse bio->bi_private as the list head. Before * ending this fs bio, we will recover its ->bi_private. */ #define REQ_DM_POLL_LIST REQ_DRV static const char *_name = DM_NAME; static unsigned int major; static unsigned int _major; static DEFINE_IDR(_minor_idr); static DEFINE_SPINLOCK(_minor_lock); static void do_deferred_remove(struct work_struct *w); static DECLARE_WORK(deferred_remove_work, do_deferred_remove); static struct workqueue_struct *deferred_remove_workqueue; atomic_t dm_global_event_nr = ATOMIC_INIT(0); DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq); void dm_issue_global_event(void) { atomic_inc(&dm_global_event_nr); wake_up(&dm_global_eventq); } DEFINE_STATIC_KEY_FALSE(stats_enabled); DEFINE_STATIC_KEY_FALSE(swap_bios_enabled); DEFINE_STATIC_KEY_FALSE(zoned_enabled); /* * One of these is allocated (on-stack) per original bio. */ struct clone_info { struct dm_table *map; struct bio *bio; struct dm_io *io; sector_t sector; unsigned int sector_count; bool is_abnormal_io:1; bool submit_as_polled:1; }; static inline struct dm_target_io *clone_to_tio(struct bio *clone) { return container_of(clone, struct dm_target_io, clone); } void *dm_per_bio_data(struct bio *bio, size_t data_size) { if (!dm_tio_flagged(clone_to_tio(bio), DM_TIO_INSIDE_DM_IO)) return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size; return (char *)bio - DM_IO_BIO_OFFSET - data_size; } EXPORT_SYMBOL_GPL(dm_per_bio_data); struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size) { struct dm_io *io = (struct dm_io *)((char *)data + data_size); if (io->magic == DM_IO_MAGIC) return (struct bio *)((char *)io + DM_IO_BIO_OFFSET); BUG_ON(io->magic != DM_TIO_MAGIC); return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET); } EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data); unsigned int dm_bio_get_target_bio_nr(const struct bio *bio) { return container_of(bio, struct dm_target_io, clone)->target_bio_nr; } EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr); #define MINOR_ALLOCED ((void *)-1) #define DM_NUMA_NODE NUMA_NO_NODE static int dm_numa_node = DM_NUMA_NODE; #define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE) static int swap_bios = DEFAULT_SWAP_BIOS; static int get_swap_bios(void) { int latch = READ_ONCE(swap_bios); if (unlikely(latch <= 0)) latch = DEFAULT_SWAP_BIOS; return latch; } struct table_device { struct list_head list; refcount_t count; struct dm_dev dm_dev; }; /* * Bio-based DM's mempools' reserved IOs set by the user. */ #define RESERVED_BIO_BASED_IOS 16 static unsigned int reserved_bio_based_ios = RESERVED_BIO_BASED_IOS; static int __dm_get_module_param_int(int *module_param, int min, int max) { int param = READ_ONCE(*module_param); int modified_param = 0; bool modified = true; if (param < min) modified_param = min; else if (param > max) modified_param = max; else modified = false; if (modified) { (void)cmpxchg(module_param, param, modified_param); param = modified_param; } return param; } unsigned int __dm_get_module_param(unsigned int *module_param, unsigned int def, unsigned int max) { unsigned int param = READ_ONCE(*module_param); unsigned int modified_param = 0; if (!param) modified_param = def; else if (param > max) modified_param = max; if (modified_param) { (void)cmpxchg(module_param, param, modified_param); param = modified_param; } return param; } unsigned int dm_get_reserved_bio_based_ios(void) { return __dm_get_module_param(&reserved_bio_based_ios, RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS); } EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios); static unsigned int dm_get_numa_node(void) { return __dm_get_module_param_int(&dm_numa_node, DM_NUMA_NODE, num_online_nodes() - 1); } static int __init local_init(void) { int r; r = dm_uevent_init(); if (r) return r; deferred_remove_workqueue = alloc_ordered_workqueue("kdmremove", 0); if (!deferred_remove_workqueue) { r = -ENOMEM; goto out_uevent_exit; } _major = major; r = register_blkdev(_major, _name); if (r < 0) goto out_free_workqueue; if (!_major) _major = r; return 0; out_free_workqueue: destroy_workqueue(deferred_remove_workqueue); out_uevent_exit: dm_uevent_exit(); return r; } static void local_exit(void) { destroy_workqueue(deferred_remove_workqueue); unregister_blkdev(_major, _name); dm_uevent_exit(); _major = 0; DMINFO("cleaned up"); } static int (*_inits[])(void) __initdata = { local_init, dm_target_init, dm_linear_init, dm_stripe_init, dm_io_init, dm_kcopyd_init, dm_interface_init, dm_statistics_init, }; static void (*_exits[])(void) = { local_exit, dm_target_exit, dm_linear_exit, dm_stripe_exit, dm_io_exit, dm_kcopyd_exit, dm_interface_exit, dm_statistics_exit, }; static int __init dm_init(void) { const int count = ARRAY_SIZE(_inits); int r, i; #if (IS_ENABLED(CONFIG_IMA) && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE)) DMINFO("CONFIG_IMA_DISABLE_HTABLE is disabled." " Duplicate IMA measurements will not be recorded in the IMA log."); #endif for (i = 0; i < count; i++) { r = _inits[i](); if (r) goto bad; } return 0; bad: while (i--) _exits[i](); return r; } static void __exit dm_exit(void) { int i = ARRAY_SIZE(_exits); while (i--) _exits[i](); /* * Should be empty by this point. */ idr_destroy(&_minor_idr); } /* * Block device functions */ int dm_deleting_md(struct mapped_device *md) { return test_bit(DMF_DELETING, &md->flags); } static int dm_blk_open(struct gendisk *disk, blk_mode_t mode) { struct mapped_device *md; spin_lock(&_minor_lock); md = disk->private_data; if (!md) goto out; if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { md = NULL; goto out; } dm_get(md); atomic_inc(&md->open_count); out: spin_unlock(&_minor_lock); return md ? 0 : -ENXIO; } static void dm_blk_close(struct gendisk *disk) { struct mapped_device *md; spin_lock(&_minor_lock); md = disk->private_data; if (WARN_ON(!md)) goto out; if (atomic_dec_and_test(&md->open_count) && (test_bit(DMF_DEFERRED_REMOVE, &md->flags))) queue_work(deferred_remove_workqueue, &deferred_remove_work); dm_put(md); out: spin_unlock(&_minor_lock); } int dm_open_count(struct mapped_device *md) { return atomic_read(&md->open_count); } /* * Guarantees nothing is using the device before it's deleted. */ int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred) { int r = 0; spin_lock(&_minor_lock); if (dm_open_count(md)) { r = -EBUSY; if (mark_deferred) set_bit(DMF_DEFERRED_REMOVE, &md->flags); } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags)) r = -EEXIST; else set_bit(DMF_DELETING, &md->flags); spin_unlock(&_minor_lock); return r; } int dm_cancel_deferred_remove(struct mapped_device *md) { int r = 0; spin_lock(&_minor_lock); if (test_bit(DMF_DELETING, &md->flags)) r = -EBUSY; else clear_bit(DMF_DEFERRED_REMOVE, &md->flags); spin_unlock(&_minor_lock); return r; } static void do_deferred_remove(struct work_struct *w) { dm_deferred_remove(); } static int dm_blk_getgeo(struct gendisk *disk, struct hd_geometry *geo) { struct mapped_device *md = disk->private_data; return dm_get_geometry(md, geo); } static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx, struct block_device **bdev, unsigned int cmd, unsigned long arg, bool *forward) { struct dm_target *ti; struct dm_table *map; int r; retry: r = -ENOTTY; map = dm_get_live_table(md, srcu_idx); if (!map || !dm_table_get_size(map)) return r; /* We only support devices that have a single target */ if (map->num_targets != 1) return r; ti = dm_table_get_target(map, 0); if (!ti->type->prepare_ioctl) return r; if (dm_suspended_md(md)) return -EAGAIN; r = ti->type->prepare_ioctl(ti, bdev, cmd, arg, forward); if (r == -ENOTCONN && *forward && !fatal_signal_pending(current)) { dm_put_live_table(md, *srcu_idx); fsleep(10000); goto retry; } return r; } static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx) { dm_put_live_table(md, srcu_idx); } static int dm_blk_ioctl(struct block_device *bdev, blk_mode_t mode, unsigned int cmd, unsigned long arg) { struct mapped_device *md = bdev->bd_disk->private_data; int r, srcu_idx; bool forward = true; r = dm_prepare_ioctl(md, &srcu_idx, &bdev, cmd, arg, &forward); if (!forward || r < 0) goto out; if (r > 0) { /* * Target determined this ioctl is being issued against a * subset of the parent bdev; require extra privileges. */ if (!capable(CAP_SYS_RAWIO)) { DMDEBUG_LIMIT( "%s: sending ioctl %x to DM device without required privilege.", current->comm, cmd); r = -ENOIOCTLCMD; goto out; } } if (!bdev->bd_disk->fops->ioctl) r = -ENOTTY; else r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg); out: dm_unprepare_ioctl(md, srcu_idx); return r; } u64 dm_start_time_ns_from_clone(struct bio *bio) { return jiffies_to_nsecs(clone_to_tio(bio)->io->start_time); } EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone); static inline unsigned int dm_io_sectors(struct dm_io *io, struct bio *bio) { /* * If REQ_PREFLUSH set, don't account payload, it will be * submitted (and accounted) after this flush completes. */ if (io->requeue_flush_with_data) return 0; if (unlikely(dm_io_flagged(io, DM_IO_WAS_SPLIT))) return io->sectors; return bio_sectors(bio); } static void dm_io_acct(struct dm_io *io, bool end) { struct bio *bio = io->orig_bio; if (dm_io_flagged(io, DM_IO_BLK_STAT)) { if (!end) bdev_start_io_acct(bio->bi_bdev, bio_op(bio), io->start_time); else bdev_end_io_acct(bio->bi_bdev, bio_op(bio), dm_io_sectors(io, bio), io->start_time); } if (static_branch_unlikely(&stats_enabled) && unlikely(dm_stats_used(&io->md->stats))) { sector_t sector; if (unlikely(dm_io_flagged(io, DM_IO_WAS_SPLIT))) sector = bio_end_sector(bio) - io->sector_offset; else sector = bio->bi_iter.bi_sector; dm_stats_account_io(&io->md->stats, bio_data_dir(bio), sector, dm_io_sectors(io, bio), end, io->start_time, &io->stats_aux); } } static void __dm_start_io_acct(struct dm_io *io) { dm_io_acct(io, false); } static void dm_start_io_acct(struct dm_io *io, struct bio *clone) { /* * Ensure IO accounting is only ever started once. */ if (dm_io_flagged(io, DM_IO_ACCOUNTED)) return; /* Expect no possibility for race unless DM_TIO_IS_DUPLICATE_BIO. */ if (!clone || likely(dm_tio_is_normal(clone_to_tio(clone)))) { dm_io_set_flag(io, DM_IO_ACCOUNTED); } else { unsigned long flags; /* Can afford locking given DM_TIO_IS_DUPLICATE_BIO */ spin_lock_irqsave(&io->lock, flags); if (dm_io_flagged(io, DM_IO_ACCOUNTED)) { spin_unlock_irqrestore(&io->lock, flags); return; } dm_io_set_flag(io, DM_IO_ACCOUNTED); spin_unlock_irqrestore(&io->lock, flags); } __dm_start_io_acct(io); } static void dm_end_io_acct(struct dm_io *io) { dm_io_acct(io, true); } static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio, gfp_t gfp_mask) { struct dm_io *io; struct dm_target_io *tio; struct bio *clone; clone = bio_alloc_clone(NULL, bio, gfp_mask, &md->mempools->io_bs); if (unlikely(!clone)) return NULL; tio = clone_to_tio(clone); tio->flags = 0; dm_tio_set_flag(tio, DM_TIO_INSIDE_DM_IO); tio->io = NULL; io = container_of(tio, struct dm_io, tio); io->magic = DM_IO_MAGIC; io->status = BLK_STS_OK; io->requeue_flush_with_data = false; /* one ref is for submission, the other is for completion */ atomic_set(&io->io_count, 2); this_cpu_inc(*md->pending_io); io->orig_bio = bio; io->md = md; spin_lock_init(&io->lock); io->start_time = jiffies; io->flags = 0; if (blk_queue_io_stat(md->queue)) dm_io_set_flag(io, DM_IO_BLK_STAT); if (static_branch_unlikely(&stats_enabled) && unlikely(dm_stats_used(&md->stats))) dm_stats_record_start(&md->stats, &io->stats_aux); return io; } static void free_io(struct dm_io *io) { bio_put(&io->tio.clone); } static struct bio *alloc_tio(struct clone_info *ci, struct dm_target *ti, unsigned int target_bio_nr, unsigned int *len, gfp_t gfp_mask) { struct mapped_device *md = ci->io->md; struct dm_target_io *tio; struct bio *clone; if (!ci->io->tio.io) { /* the dm_target_io embedded in ci->io is available */ tio = &ci->io->tio; /* alloc_io() already initialized embedded clone */ clone = &tio->clone; } else { clone = bio_alloc_clone(NULL, ci->bio, gfp_mask, &md->mempools->bs); if (!clone) return NULL; /* REQ_DM_POLL_LIST shouldn't be inherited */ clone->bi_opf &= ~REQ_DM_POLL_LIST; tio = clone_to_tio(clone); tio->flags = 0; /* also clears DM_TIO_INSIDE_DM_IO */ } tio->magic = DM_TIO_MAGIC; tio->io = ci->io; tio->ti = ti; tio->target_bio_nr = target_bio_nr; tio->len_ptr = len; tio->old_sector = 0; /* Set default bdev, but target must bio_set_dev() before issuing IO */ clone->bi_bdev = md->disk->part0; if (likely(ti != NULL) && unlikely(ti->needs_bio_set_dev)) bio_set_dev(clone, md->disk->part0); if (len) { clone->bi_iter.bi_size = to_bytes(*len); if (bio_integrity(clone)) bio_integrity_trim(clone); } return clone; } static void free_tio(struct bio *clone) { if (dm_tio_flagged(clone_to_tio(clone), DM_TIO_INSIDE_DM_IO)) return; bio_put(clone); } /* * Add the bio to the list of deferred io. */ static void queue_io(struct mapped_device *md, struct bio *bio) { unsigned long flags; spin_lock_irqsave(&md->deferred_lock, flags); bio_list_add(&md->deferred, bio); spin_unlock_irqrestore(&md->deferred_lock, flags); queue_work(md->wq, &md->work); } /* * Everyone (including functions in this file), should use this * function to access the md->map field, and make sure they call * dm_put_live_table() when finished. */ struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier) { *srcu_idx = srcu_read_lock(&md->io_barrier); return srcu_dereference(md->map, &md->io_barrier); } void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier) { srcu_read_unlock(&md->io_barrier, srcu_idx); } void dm_sync_table(struct mapped_device *md) { synchronize_srcu(&md->io_barrier); synchronize_rcu_expedited(); } /* * A fast alternative to dm_get_live_table/dm_put_live_table. * The caller must not block between these two functions. */ static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU) { rcu_read_lock(); return rcu_dereference(md->map); } static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU) { rcu_read_unlock(); } static char *_dm_claim_ptr = "I belong to device-mapper"; /* * Open a table device so we can use it as a map destination. */ static struct table_device *open_table_device(struct mapped_device *md, dev_t dev, blk_mode_t mode) { struct table_device *td; struct file *bdev_file; struct block_device *bdev; u64 part_off; int r; td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id); if (!td) return ERR_PTR(-ENOMEM); refcount_set(&td->count, 1); /* * Open the backing device with kernel rather than caller * credentials. Otherwise the caller's credentials would be * pinned in bdev_file->f_cred until the table device is closed. * That would keep the caller's thread keyring alive long beyond the * lifetime of the caller, breaking userspace expectation (e.g. * cryptsetup(8) leaking the LUKS volume key). */ scoped_with_kernel_creds() bdev_file = bdev_file_open_by_dev(dev, mode, _dm_claim_ptr, NULL); if (IS_ERR(bdev_file)) { r = PTR_ERR(bdev_file); goto out_free_td; } bdev = file_bdev(bdev_file); /* * We can be called before the dm disk is added. In that case we can't * register the holder relation here. It will be done once add_disk was * called. */ if (md->disk->slave_dir) { r = bd_link_disk_holder(bdev, md->disk); if (r) goto out_blkdev_put; } td->dm_dev.mode = mode; td->dm_dev.bdev = bdev; td->dm_dev.bdev_file = bdev_file; td->dm_dev.dax_dev = fs_dax_get_by_bdev(bdev, &part_off, NULL, NULL); format_dev_t(td->dm_dev.name, dev); list_add(&td->list, &md->table_devices); return td; out_blkdev_put: __fput_sync(bdev_file); out_free_td: kfree(td); return ERR_PTR(r); } /* * Close a table device that we've been using. */ static void close_table_device(struct table_device *td, struct mapped_device *md) { if (md->disk->slave_dir) bd_unlink_disk_holder(td->dm_dev.bdev, md->disk); /* Leverage async fput() if DMF_DEFERRED_REMOVE set */ if (unlikely(test_bit(DMF_DEFERRED_REMOVE, &md->flags))) fput(td->dm_dev.bdev_file); else __fput_sync(td->dm_dev.bdev_file); put_dax(td->dm_dev.dax_dev); list_del(&td->list); kfree(td); } static struct table_device *find_table_device(struct list_head *l, dev_t dev, blk_mode_t mode) { struct table_device *td; list_for_each_entry(td, l, list) if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode) return td; return NULL; } int dm_get_table_device(struct mapped_device *md, dev_t dev, blk_mode_t mode, struct dm_dev **result) { struct table_device *td; mutex_lock(&md->table_devices_lock); td = find_table_device(&md->table_devices, dev, mode); if (!td) { td = open_table_device(md, dev, mode); if (IS_ERR(td)) { mutex_unlock(&md->table_devices_lock); return PTR_ERR(td); } } else { refcount_inc(&td->count); } mutex_unlock(&md->table_devices_lock); *result = &td->dm_dev; return 0; } void dm_put_table_device(struct mapped_device *md, struct dm_dev *d) { struct table_device *td = container_of(d, struct table_device, dm_dev); mutex_lock(&md->table_devices_lock); if (refcount_dec_and_test(&td->count)) close_table_device(td, md); mutex_unlock(&md->table_devices_lock); } /* * Get the geometry associated with a dm device */ int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo) { *geo = md->geometry; return 0; } /* * Set the geometry of a device. */ int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo) { sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors; if (geo->start > sz) { DMERR("Start sector is beyond the geometry limits."); return -EINVAL; } md->geometry = *geo; return 0; } static int __noflush_suspending(struct mapped_device *md) { return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); } static void dm_requeue_add_io(struct dm_io *io, bool first_stage) { struct mapped_device *md = io->md; if (first_stage) { struct dm_io *next = md->requeue_list; md->requeue_list = io; io->next = next; } else { bio_list_add_head(&md->deferred, io->orig_bio); } } static void dm_kick_requeue(struct mapped_device *md, bool first_stage) { if (first_stage) queue_work(md->wq, &md->requeue_work); else queue_work(md->wq, &md->work); } /* * Return true if the dm_io's original bio is requeued. * io->status is updated with error if requeue disallowed. */ static bool dm_handle_requeue(struct dm_io *io, bool first_stage) { struct bio *bio = io->orig_bio; bool handle_requeue = (io->status == BLK_STS_DM_REQUEUE); bool handle_polled_eagain = ((io->status == BLK_STS_AGAIN) && (bio->bi_opf & REQ_POLLED)); struct mapped_device *md = io->md; bool requeued = false; if (handle_requeue || handle_polled_eagain) { unsigned long flags; if (bio->bi_opf & REQ_POLLED) { /* * Upper layer won't help us poll split bio * (io->orig_bio may only reflect a subset of the * pre-split original) so clear REQ_POLLED. */ bio_clear_polled(bio); } /* * Target requested pushing back the I/O or * polled IO hit BLK_STS_AGAIN. */ spin_lock_irqsave(&md->deferred_lock, flags); if ((__noflush_suspending(md) && !WARN_ON_ONCE(dm_is_zone_write(md, bio))) || handle_polled_eagain || first_stage) { dm_requeue_add_io(io, first_stage); requeued = true; } else { /* * noflush suspend was interrupted or this is * a write to a zoned target. */ io->status = BLK_STS_IOERR; } spin_unlock_irqrestore(&md->deferred_lock, flags); } if (requeued) dm_kick_requeue(md, first_stage); return requeued; } static void __dm_io_complete(struct dm_io *io, bool first_stage) { struct bio *bio = io->orig_bio; struct mapped_device *md = io->md; blk_status_t io_error; bool requeued; bool requeue_flush_with_data; requeued = dm_handle_requeue(io, first_stage); if (requeued && first_stage) return; io_error = io->status; if (dm_io_flagged(io, DM_IO_ACCOUNTED)) dm_end_io_acct(io); else if (!io_error) { /* * Must handle target that DM_MAPIO_SUBMITTED only to * then bio_endio() rather than dm_submit_bio_remap() */ __dm_start_io_acct(io); dm_end_io_acct(io); } requeue_flush_with_data = io->requeue_flush_with_data; free_io(io); smp_wmb(); this_cpu_dec(*md->pending_io); /* nudge anyone waiting on suspend queue */ if (unlikely(wq_has_sleeper(&md->wait))) wake_up(&md->wait); /* Return early if the original bio was requeued */ if (requeued) return; if (unlikely(requeue_flush_with_data)) { /* * Preflush done for flush with data, reissue * without REQ_PREFLUSH. */ bio->bi_opf &= ~REQ_PREFLUSH; queue_io(md, bio); } else { /* done with normal IO or empty flush */ if (io_error) bio->bi_status = io_error; bio_endio(bio); } } static void dm_wq_requeue_work(struct work_struct *work) { struct mapped_device *md = container_of(work, struct mapped_device, requeue_work); unsigned long flags; struct dm_io *io; /* reuse deferred lock to simplify dm_handle_requeue */ spin_lock_irqsave(&md->deferred_lock, flags); io = md->requeue_list; md->requeue_list = NULL; spin_unlock_irqrestore(&md->deferred_lock, flags); while (io) { struct dm_io *next = io->next; dm_io_rewind(io, &md->disk->bio_split); io->next = NULL; __dm_io_complete(io, false); io = next; cond_resched(); } } /* * Two staged requeue: * * 1) io->orig_bio points to the real original bio, and the part mapped to * this io must be requeued, instead of other parts of the original bio. * * 2) io->orig_bio points to new cloned bio which matches the requeued dm_io. */ static inline void dm_io_complete(struct dm_io *io) { /* * Only dm_io that has been split needs two stage requeue, otherwise * we may run into long bio clone chain during suspend and OOM could * be triggered. * * Also flush data dm_io won't be marked as DM_IO_WAS_SPLIT, so they * also aren't handled via the first stage requeue. */ __dm_io_complete(io, dm_io_flagged(io, DM_IO_WAS_SPLIT)); } /* * Decrements the number of outstanding ios that a bio has been * cloned into, completing the original io if necc. */ static inline void __dm_io_dec_pending(struct dm_io *io) { if (atomic_dec_and_test(&io->io_count)) dm_io_complete(io); } static void dm_io_set_error(struct dm_io *io, blk_status_t error) { unsigned long flags; /* Push-back supersedes any I/O errors */ spin_lock_irqsave(&io->lock, flags); if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(io->md))) { io->status = error; } spin_unlock_irqrestore(&io->lock, flags); } static void dm_io_dec_pending(struct dm_io *io, blk_status_t error) { if (unlikely(error)) dm_io_set_error(io, error); __dm_io_dec_pending(io); } /* * The queue_limits are only valid as long as you have a reference * count on 'md'. But _not_ imposing verification to avoid atomic_read(), */ static inline struct queue_limits *dm_get_queue_limits(struct mapped_device *md) { return &md->queue->limits; } static bool swap_bios_limit(struct dm_target *ti, struct bio *bio) { return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios); } static void clone_endio(struct bio *bio) { blk_status_t error = bio->bi_status; struct dm_target_io *tio = clone_to_tio(bio); struct dm_target *ti = tio->ti; dm_endio_fn endio = likely(ti != NULL) ? ti->type->end_io : NULL; struct dm_io *io = tio->io; struct mapped_device *md = io->md; if (unlikely(error == BLK_STS_TARGET)) { if (bio_op(bio) == REQ_OP_DISCARD && !bdev_max_discard_sectors(bio->bi_bdev)) blk_queue_disable_discard(md->queue); else if (bio_op(bio) == REQ_OP_WRITE_ZEROES && !bdev_write_zeroes_sectors(bio->bi_bdev)) blk_queue_disable_write_zeroes(md->queue); } if (static_branch_unlikely(&zoned_enabled) && unlikely(bdev_is_zoned(bio->bi_bdev))) dm_zone_endio(io, bio); if (endio) { int r = endio(ti, bio, &error); switch (r) { case DM_ENDIO_REQUEUE: if (static_branch_unlikely(&zoned_enabled)) { /* * Requeuing writes to a sequential zone of a zoned * target will break the sequential write pattern: * fail such IO. */ if (WARN_ON_ONCE(dm_is_zone_write(md, bio))) error = BLK_STS_IOERR; else error = BLK_STS_DM_REQUEUE; } else error = BLK_STS_DM_REQUEUE; fallthrough; case DM_ENDIO_DONE: break; case DM_ENDIO_INCOMPLETE: /* The target will handle the io */ return; default: DMCRIT("unimplemented target endio return value: %d", r); BUG(); } } if (static_branch_unlikely(&swap_bios_enabled) && likely(ti != NULL) && unlikely(swap_bios_limit(ti, bio))) up(&md->swap_bios_semaphore); free_tio(bio); dm_io_dec_pending(io, error); } /* * Return maximum size of I/O possible at the supplied sector up to the current * target boundary. */ static inline sector_t max_io_len_target_boundary(struct dm_target *ti, sector_t target_offset) { return ti->len - target_offset; } static sector_t __max_io_len(struct dm_target *ti, sector_t sector, unsigned int max_granularity, unsigned int max_sectors) { sector_t target_offset = dm_target_offset(ti, sector); sector_t len = max_io_len_target_boundary(ti, target_offset); /* * Does the target need to split IO even further? * - varied (per target) IO splitting is a tenet of DM; this * explains why stacked chunk_sectors based splitting via * bio_split_to_limits() isn't possible here. */ if (!max_granularity) return len; return min_t(sector_t, len, min(max_sectors ? : queue_max_sectors(ti->table->md->queue), blk_boundary_sectors_left(target_offset, max_granularity))); } static inline sector_t max_io_len(struct dm_target *ti, sector_t sector) { return __max_io_len(ti, sector, ti->max_io_len, 0); } int dm_set_target_max_io_len(struct dm_target *ti, sector_t len) { if (len > UINT_MAX) { DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)", (unsigned long long)len, UINT_MAX); ti->error = "Maximum size of target IO is too large"; return -EINVAL; } ti->max_io_len = (uint32_t) len; return 0; } EXPORT_SYMBOL_GPL(dm_set_target_max_io_len); static struct dm_target *dm_dax_get_live_target(struct mapped_device *md, sector_t sector, int *srcu_idx) __acquires(md->io_barrier) { struct dm_table *map; struct dm_target *ti; map = dm_get_live_table(md, srcu_idx); if (!map) return NULL; ti = dm_table_find_target(map, sector); if (!ti) return NULL; return ti; } static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, void **kaddr, unsigned long *pfn) { struct mapped_device *md = dax_get_private(dax_dev); sector_t sector = pgoff * PAGE_SECTORS; struct dm_target *ti; long len, ret = -EIO; int srcu_idx; ti = dm_dax_get_live_target(md, sector, &srcu_idx); if (!ti) goto out; if (!ti->type->direct_access) goto out; len = max_io_len(ti, sector) / PAGE_SECTORS; if (len < 1) goto out; nr_pages = min(len, nr_pages); ret = ti->type->direct_access(ti, pgoff, nr_pages, mode, kaddr, pfn); out: dm_put_live_table(md, srcu_idx); return ret; } static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, size_t nr_pages) { struct mapped_device *md = dax_get_private(dax_dev); sector_t sector = pgoff * PAGE_SECTORS; struct dm_target *ti; int ret = -EIO; int srcu_idx; ti = dm_dax_get_live_target(md, sector, &srcu_idx); if (!ti) goto out; if (WARN_ON(!ti->type->dax_zero_page_range)) { /* * ->zero_page_range() is mandatory dax operation. If we are * here, something is wrong. */ goto out; } ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages); out: dm_put_live_table(md, srcu_idx); return ret; } static size_t dm_dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i) { struct mapped_device *md = dax_get_private(dax_dev); sector_t sector = pgoff * PAGE_SECTORS; struct dm_target *ti; int srcu_idx; long ret = 0; ti = dm_dax_get_live_target(md, sector, &srcu_idx); if (!ti || !ti->type->dax_recovery_write) goto out; ret = ti->type->dax_recovery_write(ti, pgoff, addr, bytes, i); out: dm_put_live_table(md, srcu_idx); return ret; } /* * A target may call dm_accept_partial_bio only from the map routine. It is * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management * operations, zone append writes (native with REQ_OP_ZONE_APPEND or emulated * with write BIOs flagged with BIO_EMULATES_ZONE_APPEND) and any bio serviced * by __send_duplicate_bios(). * * dm_accept_partial_bio informs the dm that the target only wants to process * additional n_sectors sectors of the bio and the rest of the data should be * sent in a next bio. * * A diagram that explains the arithmetics: * +--------------------+---------------+-------+ * | 1 | 2 | 3 | * +--------------------+---------------+-------+ * * <-------------- *tio->len_ptr ---------------> * <----- bio_sectors -----> * <-- n_sectors --> * * Region 1 was already iterated over with bio_advance or similar function. * (it may be empty if the target doesn't use bio_advance) * Region 2 is the remaining bio size that the target wants to process. * (it may be empty if region 1 is non-empty, although there is no reason * to make it empty) * The target requires that region 3 is to be sent in the next bio. * * If the target wants to receive multiple copies of the bio (via num_*bios, etc), * the partially processed part (the sum of regions 1+2) must be the same for all * copies of the bio. */ void dm_accept_partial_bio(struct bio *bio, unsigned int n_sectors) { struct dm_target_io *tio = clone_to_tio(bio); struct dm_io *io = tio->io; unsigned int bio_sectors = bio_sectors(bio); BUG_ON(dm_tio_flagged(tio, DM_TIO_IS_DUPLICATE_BIO)); BUG_ON(bio_sectors > *tio->len_ptr); BUG_ON(n_sectors > bio_sectors); BUG_ON(bio->bi_opf & REQ_ATOMIC); if (static_branch_unlikely(&zoned_enabled) && unlikely(bdev_is_zoned(bio->bi_bdev))) { enum req_op op = bio_op(bio); BUG_ON(op_is_zone_mgmt(op)); BUG_ON(op == REQ_OP_WRITE); BUG_ON(op == REQ_OP_WRITE_ZEROES); BUG_ON(op == REQ_OP_ZONE_APPEND); } *tio->len_ptr -= bio_sectors - n_sectors; bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT; /* * __split_and_process_bio() may have already saved mapped part * for accounting but it is being reduced so update accordingly. */ dm_io_set_flag(io, DM_IO_WAS_SPLIT); io->sectors = n_sectors; io->sector_offset = bio_sectors(io->orig_bio); } EXPORT_SYMBOL_GPL(dm_accept_partial_bio); /* * @clone: clone bio that DM core passed to target's .map function * @tgt_clone: clone of @clone bio that target needs submitted * * Targets should use this interface to submit bios they take * ownership of when returning DM_MAPIO_SUBMITTED. * * Target should also enable ti->accounts_remapped_io */ void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone) { struct dm_target_io *tio = clone_to_tio(clone); struct dm_io *io = tio->io; /* establish bio that will get submitted */ if (!tgt_clone) tgt_clone = clone; bio_clone_blkg_association(tgt_clone, io->orig_bio); /* * Account io->origin_bio to DM dev on behalf of target * that took ownership of IO with DM_MAPIO_SUBMITTED. */ dm_start_io_acct(io, clone); trace_block_bio_remap(tgt_clone, disk_devt(io->md->disk), tio->old_sector); submit_bio_noacct(tgt_clone); } EXPORT_SYMBOL_GPL(dm_submit_bio_remap); static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch) { mutex_lock(&md->swap_bios_lock); while (latch < md->swap_bios) { cond_resched(); down(&md->swap_bios_semaphore); md->swap_bios--; } while (latch > md->swap_bios) { cond_resched(); up(&md->swap_bios_semaphore); md->swap_bios++; } mutex_unlock(&md->swap_bios_lock); } static void __map_bio(struct bio *clone) { struct dm_target_io *tio = clone_to_tio(clone); struct dm_target *ti = tio->ti; struct dm_io *io = tio->io; struct mapped_device *md = io->md; int r; clone->bi_end_io = clone_endio; /* * Map the clone. */ tio->old_sector = clone->bi_iter.bi_sector; if (static_branch_unlikely(&swap_bios_enabled) && unlikely(swap_bios_limit(ti, clone))) { int latch = get_swap_bios(); if (unlikely(latch != md->swap_bios)) __set_swap_bios_limit(md, latch); down(&md->swap_bios_semaphore); } if (likely(ti->type->map == linear_map)) r = linear_map(ti, clone); else if (ti->type->map == stripe_map) r = stripe_map(ti, clone); else r = ti->type->map(ti, clone); switch (r) { case DM_MAPIO_SUBMITTED: /* target has assumed ownership of this io */ if (!ti->accounts_remapped_io) dm_start_io_acct(io, clone); break; case DM_MAPIO_REMAPPED: dm_submit_bio_remap(clone, NULL); break; case DM_MAPIO_KILL: case DM_MAPIO_REQUEUE: if (static_branch_unlikely(&swap_bios_enabled) && unlikely(swap_bios_limit(ti, clone))) up(&md->swap_bios_semaphore); free_tio(clone); if (r == DM_MAPIO_KILL) dm_io_dec_pending(io, BLK_STS_IOERR); else dm_io_dec_pending(io, BLK_STS_DM_REQUEUE); break; default: DMCRIT("unimplemented target map return value: %d", r); BUG(); } } static void setup_split_accounting(struct clone_info *ci, unsigned int len) { struct dm_io *io = ci->io; if (ci->sector_count > len) { /* * Split needed, save the mapped part for accounting. * NOTE: dm_accept_partial_bio() will update accordingly. */ dm_io_set_flag(io, DM_IO_WAS_SPLIT); io->sectors = len; io->sector_offset = bio_sectors(ci->bio); } } static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci, struct dm_target *ti, unsigned int num_bios, unsigned *len) { struct bio *bio; int try; for (try = 0; try < 2; try++) { int bio_nr; if (try && num_bios > 1) mutex_lock(&ci->io->md->table_devices_lock); for (bio_nr = 0; bio_nr < num_bios; bio_nr++) { bio = alloc_tio(ci, ti, bio_nr, len, try ? GFP_NOIO : GFP_NOWAIT); if (!bio) break; bio_list_add(blist, bio); } if (try && num_bios > 1) mutex_unlock(&ci->io->md->table_devices_lock); if (bio_nr == num_bios) return; while ((bio = bio_list_pop(blist))) free_tio(bio); } } static unsigned int __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti, unsigned int num_bios, unsigned int *len) { struct bio_list blist = BIO_EMPTY_LIST; struct bio *clone; unsigned int ret = 0; if (WARN_ON_ONCE(num_bios == 0)) /* num_bios = 0 is a bug in caller */ return 0; /* dm_accept_partial_bio() is not supported with shared tio->len_ptr */ if (len) setup_split_accounting(ci, *len); /* * Using alloc_multiple_bios(), even if num_bios is 1, to consistently * support allocating using GFP_NOWAIT with GFP_NOIO fallback. */ alloc_multiple_bios(&blist, ci, ti, num_bios, len); while ((clone = bio_list_pop(&blist))) { if (num_bios > 1) dm_tio_set_flag(clone_to_tio(clone), DM_TIO_IS_DUPLICATE_BIO); __map_bio(clone); ret += 1; } return ret; } static void __send_empty_flush(struct clone_info *ci) { struct dm_table *t = ci->map; struct bio flush_bio; blk_opf_t opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC; if ((ci->io->orig_bio->bi_opf & (REQ_IDLE | REQ_SYNC)) == (REQ_IDLE | REQ_SYNC)) opf |= REQ_IDLE; /* * Use an on-stack bio for this, it's safe since we don't * need to reference it after submit. It's just used as * the basis for the clone(s). */ bio_init(&flush_bio, ci->io->md->disk->part0, NULL, 0, opf); ci->bio = &flush_bio; ci->sector_count = 0; ci->io->tio.clone.bi_iter.bi_size = 0; if (!t->flush_bypasses_map) { for (unsigned int i = 0; i < t->num_targets; i++) { unsigned int bios; struct dm_target *ti = dm_table_get_target(t, i); if (unlikely(ti->num_flush_bios == 0)) continue; atomic_add(ti->num_flush_bios, &ci->io->io_count); bios = __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL); atomic_sub(ti->num_flush_bios - bios, &ci->io->io_count); } } else { /* * Note that there's no need to grab t->devices_lock here * because the targets that support flush optimization don't * modify the list of devices. */ struct list_head *devices = dm_table_get_devices(t); unsigned int len = 0; struct dm_dev_internal *dd; list_for_each_entry(dd, devices, list) { struct bio *clone; /* * Note that the structure dm_target_io is not * associated with any target (because the device may be * used by multiple targets), so we set tio->ti = NULL. * We must check for NULL in the I/O processing path, to * avoid NULL pointer dereference. */ clone = alloc_tio(ci, NULL, 0, &len, GFP_NOIO); atomic_add(1, &ci->io->io_count); bio_set_dev(clone, dd->dm_dev->bdev); clone->bi_end_io = clone_endio; dm_submit_bio_remap(clone, NULL); } } /* * alloc_io() takes one extra reference for submission, so the * reference won't reach 0 without the following subtraction */ atomic_sub(1, &ci->io->io_count); bio_uninit(ci->bio); } static void __send_abnormal_io(struct clone_info *ci, struct dm_target *ti, unsigned int num_bios, unsigned int max_granularity, unsigned int max_sectors) { unsigned int len, bios; len = min_t(sector_t, ci->sector_count, __max_io_len(ti, ci->sector, max_granularity, max_sectors)); atomic_add(num_bios, &ci->io->io_count); bios = __send_duplicate_bios(ci, ti, num_bios, &len); /* * alloc_io() takes one extra reference for submission, so the * reference won't reach 0 without the following (+1) subtraction */ atomic_sub(num_bios - bios + 1, &ci->io->io_count); ci->sector += len; ci->sector_count -= len; } static bool is_abnormal_io(struct bio *bio) { switch (bio_op(bio)) { case REQ_OP_READ: case REQ_OP_WRITE: case REQ_OP_FLUSH: return false; case REQ_OP_DISCARD: case REQ_OP_SECURE_ERASE: case REQ_OP_WRITE_ZEROES: case REQ_OP_ZONE_RESET_ALL: return true; default: return false; } } static blk_status_t __process_abnormal_io(struct clone_info *ci, struct dm_target *ti) { unsigned int num_bios = 0; unsigned int max_granularity = 0; unsigned int max_sectors = 0; struct queue_limits *limits = dm_get_queue_limits(ti->table->md); switch (bio_op(ci->bio)) { case REQ_OP_DISCARD: num_bios = ti->num_discard_bios; max_sectors = limits->max_discard_sectors; if (ti->max_discard_granularity) max_granularity = max_sectors; break; case REQ_OP_SECURE_ERASE: num_bios = ti->num_secure_erase_bios; max_sectors = limits->max_secure_erase_sectors; break; case REQ_OP_WRITE_ZEROES: num_bios = ti->num_write_zeroes_bios; max_sectors = limits->max_write_zeroes_sectors; break; default: break; } /* * Even though the device advertised support for this type of * request, that does not mean every target supports it, and * reconfiguration might also have changed that since the * check was performed. */ if (unlikely(!num_bios)) return BLK_STS_NOTSUPP; __send_abnormal_io(ci, ti, num_bios, max_granularity, max_sectors); return BLK_STS_OK; } /* * Reuse ->bi_private as dm_io list head for storing all dm_io instances * associated with this bio, and this bio's bi_private needs to be * stored in dm_io->data before the reuse. * * bio->bi_private is owned by fs or upper layer, so block layer won't * touch it after splitting. Meantime it won't be changed by anyone after * bio is submitted. So this reuse is safe. */ static inline struct dm_io **dm_poll_list_head(struct bio *bio) { return (struct dm_io **)&bio->bi_private; } static void dm_queue_poll_io(struct bio *bio, struct dm_io *io) { struct dm_io **head = dm_poll_list_head(bio); if (!(bio->bi_opf & REQ_DM_POLL_LIST)) { bio->bi_opf |= REQ_DM_POLL_LIST; /* * Save .bi_private into dm_io, so that we can reuse * .bi_private as dm_io list head for storing dm_io list */ io->data = bio->bi_private; /* tell block layer to poll for completion */ bio->bi_cookie = ~BLK_QC_T_NONE; io->next = NULL; } else { /* * bio recursed due to split, reuse original poll list, * and save bio->bi_private too. */ io->data = (*head)->data; io->next = *head; } *head = io; } /* * Select the correct strategy for processing a non-flush bio. */ static blk_status_t __split_and_process_bio(struct clone_info *ci) { struct bio *clone; struct dm_target *ti; unsigned int len; ti = dm_table_find_target(ci->map, ci->sector); if (unlikely(!ti)) return BLK_STS_IOERR; if (unlikely(ci->is_abnormal_io)) return __process_abnormal_io(ci, ti); /* * Only support bio polling for normal IO, and the target io is * exactly inside the dm_io instance (verified in dm_poll_dm_io) */ ci->submit_as_polled = !!(ci->bio->bi_opf & REQ_POLLED); len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count); if (ci->bio->bi_opf & REQ_ATOMIC) { if (unlikely(!dm_target_supports_atomic_writes(ti->type))) return BLK_STS_IOERR; if (unlikely(len != ci->sector_count)) return BLK_STS_IOERR; } setup_split_accounting(ci, len); if (unlikely(ci->bio->bi_opf & REQ_NOWAIT)) { if (unlikely(!dm_target_supports_nowait(ti->type))) return BLK_STS_NOTSUPP; clone = alloc_tio(ci, ti, 0, &len, GFP_NOWAIT); if (unlikely(!clone)) return BLK_STS_AGAIN; } else { clone = alloc_tio(ci, ti, 0, &len, GFP_NOIO); } __map_bio(clone); ci->sector += len; ci->sector_count -= len; return BLK_STS_OK; } static void init_clone_info(struct clone_info *ci, struct dm_io *io, struct dm_table *map, struct bio *bio, bool is_abnormal) { ci->map = map; ci->io = io; ci->bio = bio; ci->is_abnormal_io = is_abnormal; ci->submit_as_polled = false; ci->sector = bio->bi_iter.bi_sector; ci->sector_count = bio_sectors(bio); /* Shouldn't happen but sector_count was being set to 0 so... */ if (static_branch_unlikely(&zoned_enabled) && WARN_ON_ONCE(op_is_zone_mgmt(bio_op(bio)) && ci->sector_count)) ci->sector_count = 0; } #ifdef CONFIG_BLK_DEV_ZONED static inline bool dm_zone_bio_needs_split(struct bio *bio) { /* * Special case the zone operations that cannot or should not be split. */ switch (bio_op(bio)) { case REQ_OP_ZONE_APPEND: case REQ_OP_ZONE_FINISH: case REQ_OP_ZONE_RESET: case REQ_OP_ZONE_RESET_ALL: return false; default: break; } /* * When mapped devices use the block layer zone write plugging, we must * split any large BIO to the mapped device limits to not submit BIOs * that span zone boundaries and to avoid potential deadlocks with * queue freeze operations. */ return bio_needs_zone_write_plugging(bio) || bio_straddles_zones(bio); } static inline bool dm_zone_plug_bio(struct mapped_device *md, struct bio *bio) { if (!bio_needs_zone_write_plugging(bio)) return false; return blk_zone_plug_bio(bio, 0); } static blk_status_t __send_zone_reset_all_emulated(struct clone_info *ci, struct dm_target *ti) { struct bio_list blist = BIO_EMPTY_LIST; struct mapped_device *md = ci->io->md; unsigned int zone_sectors = md->disk->queue->limits.chunk_sectors; unsigned long *need_reset; unsigned int i, nr_zones, nr_reset; unsigned int num_bios = 0; blk_status_t sts = BLK_STS_OK; sector_t sector = ti->begin; struct bio *clone; int ret; nr_zones = ti->len >> ilog2(zone_sectors); need_reset = bitmap_zalloc(nr_zones, GFP_NOIO); if (!need_reset) return BLK_STS_RESOURCE; ret = dm_zone_get_reset_bitmap(md, ci->map, ti->begin, nr_zones, need_reset); if (ret) { sts = BLK_STS_IOERR; goto free_bitmap; } /* If we have no zone to reset, we are done. */ nr_reset = bitmap_weight(need_reset, nr_zones); if (!nr_reset) goto free_bitmap; atomic_add(nr_zones, &ci->io->io_count); for (i = 0; i < nr_zones; i++) { if (!test_bit(i, need_reset)) { sector += zone_sectors; continue; } if (bio_list_empty(&blist)) { /* This may take a while, so be nice to others */ if (num_bios) cond_resched(); /* * We may need to reset thousands of zones, so let's * not go crazy with the clone allocation. */ alloc_multiple_bios(&blist, ci, ti, min(nr_reset, 32), NULL); } /* Get a clone and change it to a regular reset operation. */ clone = bio_list_pop(&blist); clone->bi_opf &= ~REQ_OP_MASK; clone->bi_opf |= REQ_OP_ZONE_RESET | REQ_SYNC; clone->bi_iter.bi_sector = sector; clone->bi_iter.bi_size = 0; __map_bio(clone); sector += zone_sectors; num_bios++; nr_reset--; } WARN_ON_ONCE(!bio_list_empty(&blist)); atomic_sub(nr_zones - num_bios, &ci->io->io_count); ci->sector_count = 0; free_bitmap: bitmap_free(need_reset); return sts; } static void __send_zone_reset_all_native(struct clone_info *ci, struct dm_target *ti) { unsigned int bios; atomic_add(1, &ci->io->io_count); bios = __send_duplicate_bios(ci, ti, 1, NULL); atomic_sub(1 - bios, &ci->io->io_count); ci->sector_count = 0; } static blk_status_t __send_zone_reset_all(struct clone_info *ci) { struct dm_table *t = ci->map; blk_status_t sts = BLK_STS_OK; for (unsigned int i = 0; i < t->num_targets; i++) { struct dm_target *ti = dm_table_get_target(t, i); if (ti->zone_reset_all_supported) { __send_zone_reset_all_native(ci, ti); continue; } sts = __send_zone_reset_all_emulated(ci, ti); if (sts != BLK_STS_OK) break; } /* Release the reference that alloc_io() took for submission. */ atomic_sub(1, &ci->io->io_count); return sts; } #else static inline bool dm_zone_bio_needs_split(struct bio *bio) { return false; } static inline bool dm_zone_plug_bio(struct mapped_device *md, struct bio *bio) { return false; } static blk_status_t __send_zone_reset_all(struct clone_info *ci) { return BLK_STS_NOTSUPP; } #endif /* * Entry point to split a bio into clones and submit them to the targets. */ static void dm_split_and_process_bio(struct mapped_device *md, struct dm_table *map, struct bio *bio) { struct clone_info ci; struct dm_io *io; blk_status_t error = BLK_STS_OK; bool is_abnormal, need_split; is_abnormal = is_abnormal_io(bio); if (static_branch_unlikely(&zoned_enabled)) { need_split = is_abnormal || dm_zone_bio_needs_split(bio); } else { need_split = is_abnormal; } if (unlikely(need_split)) { /* * Use bio_split_to_limits() for abnormal IO (e.g. discard, etc) * otherwise associated queue_limits won't be imposed. * Also split the BIO for mapped devices needing zone append * emulation to ensure that the BIO does not cross zone * boundaries. */ bio = bio_split_to_limits(bio); if (!bio) return; } /* * Use the block layer zone write plugging for mapped devices that * need zone append emulation (e.g. dm-crypt). */ if (static_branch_unlikely(&zoned_enabled) && dm_zone_plug_bio(md, bio)) return; /* Only support nowait for normal IO */ if (unlikely(bio->bi_opf & REQ_NOWAIT) && !is_abnormal) { /* * Don't support NOWAIT for FLUSH because it may allocate * multiple bios and there's no easy way how to undo the * allocations. */ if (bio->bi_opf & REQ_PREFLUSH) { bio_wouldblock_error(bio); return; } io = alloc_io(md, bio, GFP_NOWAIT); if (unlikely(!io)) { /* Unable to do anything without dm_io. */ bio_wouldblock_error(bio); return; } } else { io = alloc_io(md, bio, GFP_NOIO); } init_clone_info(&ci, io, map, bio, is_abnormal); if (unlikely((bio->bi_opf & REQ_PREFLUSH) != 0)) { /* * The "flush_bypasses_map" is set on targets where it is safe * to skip the map function and submit bios directly to the * underlying block devices - currently, it is set for dm-linear * and dm-stripe. * * If we have just one underlying device (i.e. there is one * linear target or multiple linear targets pointing to the same * device), we can send the flush with data directly to it. */ if (bio->bi_iter.bi_size && map->flush_bypasses_map) { struct list_head *devices = dm_table_get_devices(map); if (devices->next == devices->prev) goto send_preflush_with_data; } if (bio->bi_iter.bi_size) io->requeue_flush_with_data = true; __send_empty_flush(&ci); /* dm_io_complete submits any data associated with flush */ goto out; } send_preflush_with_data: if (static_branch_unlikely(&zoned_enabled) && (bio_op(bio) == REQ_OP_ZONE_RESET_ALL)) { error = __send_zone_reset_all(&ci); goto out; } error = __split_and_process_bio(&ci); if (error || !ci.sector_count) goto out; /* * Remainder must be passed to submit_bio_noacct() so it gets handled * *after* bios already submitted have been completely processed. */ bio_trim(bio, io->sectors, ci.sector_count); trace_block_split(bio, bio->bi_iter.bi_sector); bio_inc_remaining(bio); submit_bio_noacct(bio); out: /* * Drop the extra reference count for non-POLLED bio, and hold one * reference for POLLED bio, which will be released in dm_poll_bio * * Add every dm_io instance into the dm_io list head which is stored * in bio->bi_private, so that dm_poll_bio can poll them all. */ if (error || !ci.submit_as_polled) { /* * In case of submission failure, the extra reference for * submitting io isn't consumed yet */ if (error) atomic_dec(&io->io_count); dm_io_dec_pending(io, error); } else dm_queue_poll_io(bio, io); } static void dm_submit_bio(struct bio *bio) { struct mapped_device *md = bio->bi_bdev->bd_disk->private_data; int srcu_idx; struct dm_table *map; map = dm_get_live_table(md, &srcu_idx); if (unlikely(!map)) { DMERR_LIMIT("%s: mapping table unavailable, erroring io", dm_device_name(md)); bio_io_error(bio); goto out; } /* If suspended, queue this IO for later */ if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) { if (bio->bi_opf & REQ_NOWAIT) bio_wouldblock_error(bio); else if (bio->bi_opf & REQ_RAHEAD) bio_io_error(bio); else queue_io(md, bio); goto out; } dm_split_and_process_bio(md, map, bio); out: dm_put_live_table(md, srcu_idx); } static bool dm_poll_dm_io(struct dm_io *io, struct io_comp_batch *iob, unsigned int flags) { WARN_ON_ONCE(!dm_tio_is_normal(&io->tio)); /* don't poll if the mapped io is done */ if (atomic_read(&io->io_count) > 1) { /* * DM hides the target queues from the upper poller, which may * decide it is safe to spin on a single stacked queue. Do not * pass that spinning policy down to a target queue: one slow * clone could keep the task inside dm_poll_bio() for a long * time. Poll target bios once and let the caller decide * whether to keep polling, reap completions or reschedule. */ bio_poll(&io->tio.clone, iob, flags | BLK_POLL_ONESHOT); } /* bio_poll holds the last reference */ return atomic_read(&io->io_count) == 1; } static int dm_poll_bio(struct bio *bio, struct io_comp_batch *iob, unsigned int flags) { struct dm_io **head = dm_poll_list_head(bio); struct dm_io *list = *head; struct dm_io *tmp = NULL; struct dm_io *curr, *next; /* Only poll normal bio which was marked as REQ_DM_POLL_LIST */ if (!(bio->bi_opf & REQ_DM_POLL_LIST)) return 0; WARN_ON_ONCE(!list); /* * Restore .bi_private before possibly completing dm_io. * * bio_poll() is only possible once @bio has been completely * submitted via submit_bio_noacct()'s depth-first submission. * So there is no dm_queue_poll_io() race associated with * clearing REQ_DM_POLL_LIST here. */ bio->bi_opf &= ~REQ_DM_POLL_LIST; bio->bi_private = list->data; for (curr = list, next = curr->next; curr; curr = next, next = curr ? curr->next : NULL) { if (dm_poll_dm_io(curr, iob, flags)) { /* * clone_endio() has already occurred, so no * error handling is needed here. */ __dm_io_dec_pending(curr); } else { curr->next = tmp; tmp = curr; } } /* Not done? */ if (tmp) { bio->bi_opf |= REQ_DM_POLL_LIST; /* Reset bio->bi_private to dm_io list head */ *head = tmp; return 0; } return 1; } /* *--------------------------------------------------------------- * An IDR is used to keep track of allocated minor numbers. *--------------------------------------------------------------- */ static void free_minor(int minor) { spin_lock(&_minor_lock); idr_remove(&_minor_idr, minor); spin_unlock(&_minor_lock); } /* * See if the device with a specific minor # is free. */ static int specific_minor(int minor) { int r; if (minor >= (1 << MINORBITS)) return -EINVAL; idr_preload(GFP_KERNEL); spin_lock(&_minor_lock); r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT); spin_unlock(&_minor_lock); idr_preload_end(); if (r < 0) return r == -ENOSPC ? -EBUSY : r; return 0; } static int next_free_minor(int *minor) { int r; idr_preload(GFP_KERNEL); spin_lock(&_minor_lock); r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT); spin_unlock(&_minor_lock); idr_preload_end(); if (r < 0) return r; *minor = r; return 0; } static const struct block_device_operations dm_blk_dops; static const struct block_device_operations dm_rq_blk_dops; static const struct dax_operations dm_dax_ops; static void dm_wq_work(struct work_struct *work); #ifdef CONFIG_BLK_INLINE_ENCRYPTION static void dm_queue_destroy_crypto_profile(struct request_queue *q) { dm_destroy_crypto_profile(q->crypto_profile); } #else /* CONFIG_BLK_INLINE_ENCRYPTION */ static inline void dm_queue_destroy_crypto_profile(struct request_queue *q) { } #endif /* !CONFIG_BLK_INLINE_ENCRYPTION */ static void cleanup_mapped_device(struct mapped_device *md) { if (md->wq) destroy_workqueue(md->wq); dm_free_md_mempools(md->mempools); if (md->dax_dev) { dax_remove_host(md->disk); kill_dax(md->dax_dev); put_dax(md->dax_dev); md->dax_dev = NULL; } if (md->disk) { spin_lock(&_minor_lock); md->disk->private_data = NULL; spin_unlock(&_minor_lock); if (dm_get_md_type(md) != DM_TYPE_NONE) { struct table_device *td; dm_sysfs_exit(md); list_for_each_entry(td, &md->table_devices, list) { bd_unlink_disk_holder(td->dm_dev.bdev, md->disk); } /* * Hold lock to make sure del_gendisk() won't concurrent * with open/close_table_device(). */ mutex_lock(&md->table_devices_lock); del_gendisk(md->disk); mutex_unlock(&md->table_devices_lock); } dm_queue_destroy_crypto_profile(md->queue); put_disk(md->disk); } if (md->pending_io) { free_percpu(md->pending_io); md->pending_io = NULL; } cleanup_srcu_struct(&md->io_barrier); mutex_destroy(&md->suspend_lock); mutex_destroy(&md->type_lock); mutex_destroy(&md->table_devices_lock); mutex_destroy(&md->swap_bios_lock); dm_mq_cleanup_mapped_device(md); } /* * Allocate and initialise a blank device with a given minor. */ static struct mapped_device *alloc_dev(int minor) { int r, numa_node_id = dm_get_numa_node(); struct dax_device *dax_dev; struct mapped_device *md; void *old_md; md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id); if (!md) { DMERR("unable to allocate device, out of memory."); return NULL; } if (!try_module_get(THIS_MODULE)) goto bad_module_get; /* get a minor number for the dev */ if (minor == DM_ANY_MINOR) r = next_free_minor(&minor); else r = specific_minor(minor); if (r < 0) goto bad_minor; r = init_srcu_struct(&md->io_barrier); if (r < 0) goto bad_io_barrier; md->numa_node_id = numa_node_id; md->init_tio_pdu = false; md->type = DM_TYPE_NONE; mutex_init(&md->suspend_lock); mutex_init(&md->type_lock); mutex_init(&md->table_devices_lock); spin_lock_init(&md->deferred_lock); atomic_set(&md->holders, 1); atomic_set(&md->open_count, 0); atomic_set(&md->event_nr, 0); atomic_set(&md->uevent_seq, 0); INIT_LIST_HEAD(&md->uevent_list); INIT_LIST_HEAD(&md->table_devices); spin_lock_init(&md->uevent_lock); /* * default to bio-based until DM table is loaded and md->type * established. If request-based table is loaded: blk-mq will * override accordingly. */ md->disk = blk_alloc_disk(NULL, md->numa_node_id); if (IS_ERR(md->disk)) { md->disk = NULL; goto bad; } md->queue = md->disk->queue; init_waitqueue_head(&md->wait); INIT_WORK(&md->work, dm_wq_work); INIT_WORK(&md->requeue_work, dm_wq_requeue_work); init_waitqueue_head(&md->eventq); init_completion(&md->kobj_holder.completion); md->requeue_list = NULL; md->swap_bios = get_swap_bios(); sema_init(&md->swap_bios_semaphore, md->swap_bios); mutex_init(&md->swap_bios_lock); md->disk->major = _major; md->disk->first_minor = minor; md->disk->minors = 1; md->disk->flags |= GENHD_FL_NO_PART; md->disk->fops = &dm_blk_dops; md->disk->private_data = md; sprintf(md->disk->disk_name, "dm-%d", minor); dax_dev = alloc_dax(md, &dm_dax_ops); if (IS_ERR(dax_dev)) { if (PTR_ERR(dax_dev) != -EOPNOTSUPP) goto bad; } else { set_dax_nocache(dax_dev); set_dax_nomc(dax_dev); md->dax_dev = dax_dev; if (dax_add_host(dax_dev, md->disk)) goto bad; } format_dev_t(md->name, MKDEV(_major, minor)); md->wq = alloc_workqueue("kdmflush/%s", WQ_MEM_RECLAIM | WQ_PERCPU, 0, md->name); if (!md->wq) goto bad; md->pending_io = alloc_percpu(unsigned long); if (!md->pending_io) goto bad; r = dm_stats_init(&md->stats); if (r < 0) goto bad; /* Populate the mapping, nobody knows we exist yet */ spin_lock(&_minor_lock); old_md = idr_replace(&_minor_idr, md, minor); spin_unlock(&_minor_lock); BUG_ON(old_md != MINOR_ALLOCED); return md; bad: cleanup_mapped_device(md); bad_io_barrier: free_minor(minor); bad_minor: module_put(THIS_MODULE); bad_module_get: kvfree(md); return NULL; } static void unlock_fs(struct mapped_device *md); static void free_dev(struct mapped_device *md) { int minor = MINOR(disk_devt(md->disk)); unlock_fs(md); cleanup_mapped_device(md); WARN_ON_ONCE(!list_empty(&md->table_devices)); dm_stats_cleanup(&md->stats); free_minor(minor); module_put(THIS_MODULE); kvfree(md); } /* * Bind a table to the device. */ static void event_callback(void *context) { unsigned long flags; LIST_HEAD(uevents); struct mapped_device *md = context; spin_lock_irqsave(&md->uevent_lock, flags); list_splice_init(&md->uevent_list, &uevents); spin_unlock_irqrestore(&md->uevent_lock, flags); dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj); atomic_inc(&md->event_nr); wake_up(&md->eventq); dm_issue_global_event(); } /* * Returns old map, which caller must destroy. */ static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t, struct queue_limits *limits) { struct dm_table *old_map; sector_t size, old_size; lockdep_assert_held(&md->suspend_lock); size = dm_table_get_size(t); old_size = dm_get_size(md); if (!dm_table_supports_size_change(t, old_size, size)) { old_map = ERR_PTR(-EINVAL); goto out; } set_capacity(md->disk, size); if (limits) { int ret = dm_table_set_restrictions(t, md->queue, limits); if (ret) { set_capacity(md->disk, old_size); old_map = ERR_PTR(ret); goto out; } } /* * Wipe any geometry if the size of the table changed. */ if (size != old_size) memset(&md->geometry, 0, sizeof(md->geometry)); dm_table_event_callback(t, event_callback, md); if (dm_table_request_based(t)) { /* * Leverage the fact that request-based DM targets are * immutable singletons - used to optimize dm_mq_queue_rq. */ md->immutable_target = dm_table_get_immutable_target(t); /* * There is no need to reload with request-based dm because the * size of front_pad doesn't change. * * Note for future: If you are to reload bioset, prep-ed * requests in the queue may refer to bio from the old bioset, * so you must walk through the queue to unprep. */ if (!md->mempools) md->mempools = t->mempools; else dm_free_md_mempools(t->mempools); } else { /* * The md may already have mempools that need changing. * If so, reload bioset because front_pad may have changed * because a different table was loaded. */ dm_free_md_mempools(md->mempools); md->mempools = t->mempools; } t->mempools = NULL; old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); rcu_assign_pointer(md->map, (void *)t); md->immutable_target_type = dm_table_get_immutable_target_type(t); if (old_map) dm_sync_table(md); out: return old_map; } /* * Returns unbound table for the caller to free. */ static struct dm_table *__unbind(struct mapped_device *md) { struct dm_table *map = rcu_dereference_protected(md->map, 1); if (!map) return NULL; dm_table_event_callback(map, NULL, NULL); RCU_INIT_POINTER(md->map, NULL); dm_sync_table(md); return map; } /* * Constructor for a new device. */ int dm_create(int minor, struct mapped_device **result) { struct mapped_device *md; md = alloc_dev(minor); if (!md) return -ENXIO; dm_ima_init(md); *result = md; return 0; } /* * Functions to manage md->type. * All are required to hold md->type_lock. */ void dm_lock_md_type(struct mapped_device *md) { mutex_lock(&md->type_lock); } void dm_unlock_md_type(struct mapped_device *md) { mutex_unlock(&md->type_lock); } enum dm_queue_mode dm_get_md_type(struct mapped_device *md) { return md->type; } struct target_type *dm_get_immutable_target_type(struct mapped_device *md) { return md->immutable_target_type; } /* * Setup the DM device's queue based on md's type */ int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t) { enum dm_queue_mode type = dm_table_get_type(t); struct queue_limits limits; struct table_device *td; int r; WARN_ON_ONCE(type == DM_TYPE_NONE); if (type == DM_TYPE_REQUEST_BASED) { md->disk->fops = &dm_rq_blk_dops; r = dm_mq_init_request_queue(md, t); if (r) { DMERR("Cannot initialize queue for request-based dm mapped device"); return r; } } r = dm_calculate_queue_limits(t, &limits); if (r) { DMERR("Cannot calculate initial queue limits"); return r; } r = dm_table_set_restrictions(t, md->queue, &limits); if (r) return r; /* * Hold lock to make sure add_disk() and del_gendisk() won't concurrent * with open_table_device() and close_table_device(). */ mutex_lock(&md->table_devices_lock); r = add_disk(md->disk); mutex_unlock(&md->table_devices_lock); if (r) return r; /* * Register the holder relationship for devices added before the disk * was live. */ list_for_each_entry(td, &md->table_devices, list) { r = bd_link_disk_holder(td->dm_dev.bdev, md->disk); if (r) goto out_undo_holders; } r = dm_sysfs_init(md); if (r) goto out_undo_holders; md->type = type; return 0; out_undo_holders: list_for_each_entry_continue_reverse(td, &md->table_devices, list) bd_unlink_disk_holder(td->dm_dev.bdev, md->disk); mutex_lock(&md->table_devices_lock); del_gendisk(md->disk); mutex_unlock(&md->table_devices_lock); return r; } struct mapped_device *dm_get_md(dev_t dev) { struct mapped_device *md; unsigned int minor = MINOR(dev); if (MAJOR(dev) != _major || minor >= (1 << MINORBITS)) return NULL; spin_lock(&_minor_lock); md = idr_find(&_minor_idr, minor); if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) || test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { md = NULL; goto out; } dm_get(md); out: spin_unlock(&_minor_lock); return md; } EXPORT_SYMBOL_GPL(dm_get_md); void *dm_get_mdptr(struct mapped_device *md) { return md->interface_ptr; } void dm_set_mdptr(struct mapped_device *md, void *ptr) { md->interface_ptr = ptr; } void dm_get(struct mapped_device *md) { atomic_inc(&md->holders); BUG_ON(test_bit(DMF_FREEING, &md->flags)); } int dm_hold(struct mapped_device *md) { spin_lock(&_minor_lock); if (test_bit(DMF_FREEING, &md->flags)) { spin_unlock(&_minor_lock); return -EBUSY; } dm_get(md); spin_unlock(&_minor_lock); return 0; } EXPORT_SYMBOL_GPL(dm_hold); const char *dm_device_name(struct mapped_device *md) { return md->name; } EXPORT_SYMBOL_GPL(dm_device_name); static void __dm_destroy(struct mapped_device *md, bool wait) { struct dm_table *map; int srcu_idx; might_sleep(); spin_lock(&_minor_lock); idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md)))); set_bit(DMF_FREEING, &md->flags); spin_unlock(&_minor_lock); blk_mark_disk_dead(md->disk); /* * Take suspend_lock so that presuspend and postsuspend methods * do not race with internal suspend. */ mutex_lock(&md->suspend_lock); map = dm_get_live_table(md, &srcu_idx); if (!dm_suspended_md(md)) { dm_table_presuspend_targets(map); set_bit(DMF_SUSPENDED, &md->flags); set_bit(DMF_POST_SUSPENDING, &md->flags); dm_table_postsuspend_targets(map); } /* dm_put_live_table must be before fsleep, otherwise deadlock is possible */ dm_put_live_table(md, srcu_idx); mutex_unlock(&md->suspend_lock); /* * Rare, but there may be I/O requests still going to complete, * for example. Wait for all references to disappear. * No one should increment the reference count of the mapped_device, * after the mapped_device state becomes DMF_FREEING. */ if (wait) while (atomic_read(&md->holders)) fsleep(1000); else if (atomic_read(&md->holders)) DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)", dm_device_name(md), atomic_read(&md->holders)); dm_table_destroy(__unbind(md)); free_dev(md); } void dm_destroy(struct mapped_device *md) { __dm_destroy(md, true); } void dm_destroy_immediate(struct mapped_device *md) { __dm_destroy(md, false); } void dm_put(struct mapped_device *md) { atomic_dec(&md->holders); } EXPORT_SYMBOL_GPL(dm_put); static bool dm_in_flight_bios(struct mapped_device *md) { int cpu; unsigned long sum = 0; for_each_possible_cpu(cpu) sum += *per_cpu_ptr(md->pending_io, cpu); return sum != 0; } static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state) { int r = 0; DEFINE_WAIT(wait); while (true) { prepare_to_wait(&md->wait, &wait, task_state); if (!dm_in_flight_bios(md)) break; if (signal_pending_state(task_state, current)) { r = -ERESTARTSYS; break; } io_schedule(); } finish_wait(&md->wait, &wait); smp_rmb(); return r; } static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state) { int r = 0; if (!queue_is_mq(md->queue)) return dm_wait_for_bios_completion(md, task_state); while (true) { if (!blk_mq_queue_inflight(md->queue)) break; if (signal_pending_state(task_state, current)) { r = -ERESTARTSYS; break; } fsleep(5000); } return r; } /* * Process the deferred bios */ static void dm_wq_work(struct work_struct *work) { struct mapped_device *md = container_of(work, struct mapped_device, work); struct bio *bio; while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) { spin_lock_irq(&md->deferred_lock); bio = bio_list_pop(&md->deferred); spin_unlock_irq(&md->deferred_lock); if (!bio) break; submit_bio_noacct(bio); cond_resched(); } } static void dm_queue_flush(struct mapped_device *md) { clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); smp_mb__after_atomic(); queue_work(md->wq, &md->work); } /* * Swap in a new table, returning the old one for the caller to destroy. */ struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table) { struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL); struct queue_limits limits; bool update_limits = true; int r; mutex_lock(&md->suspend_lock); /* device must be suspended */ if (!dm_suspended_md(md)) goto out; /* * To avoid a potential deadlock locking the queue limits, disallow * updating the queue limits during a table swap, when updating an * immutable request-based dm device (dm-multipath) during a noflush * suspend. It is userspace's responsibility to make sure that the new * table uses the same limits as the existing table, if it asks for a * noflush suspend. */ if (dm_request_based(md) && md->immutable_target && __noflush_suspending(md)) update_limits = false; /* * If the new table has no data devices, retain the existing limits. * This helps multipath with queue_if_no_path if all paths disappear, * then new I/O is queued based on these limits, and then some paths * reappear. */ else if (dm_table_has_no_data_devices(table)) { live_map = dm_get_live_table_fast(md); if (live_map) limits = md->queue->limits; dm_put_live_table_fast(md); } if (update_limits && !live_map) { r = dm_calculate_queue_limits(table, &limits); if (r) { map = ERR_PTR(r); goto out; } } map = __bind(md, table, update_limits ? &limits : NULL); dm_issue_global_event(); out: mutex_unlock(&md->suspend_lock); return map; } /* * Functions to lock and unlock any filesystem running on the * device. */ static int lock_fs(struct mapped_device *md) { int r; WARN_ON(test_bit(DMF_FROZEN, &md->flags)); r = bdev_freeze(md->disk->part0); if (!r) set_bit(DMF_FROZEN, &md->flags); return r; } static void unlock_fs(struct mapped_device *md) { if (!test_bit(DMF_FROZEN, &md->flags)) return; bdev_thaw(md->disk->part0); clear_bit(DMF_FROZEN, &md->flags); } /* * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY * * If __dm_suspend returns 0, the device is completely quiescent * now. There is no request-processing activity. All new requests * are being added to md->deferred list. */ static int __dm_suspend(struct mapped_device *md, struct dm_table *map, unsigned int suspend_flags, unsigned int task_state, int dmf_suspended_flag) { bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG; bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG; int r = 0; lockdep_assert_held(&md->suspend_lock); /* * DMF_NOFLUSH_SUSPENDING must be set before presuspend. */ if (noflush) set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags); else DMDEBUG("%s: suspending with flush", dm_device_name(md)); /* * This gets reverted if there's an error later and the targets * provide the .presuspend_undo hook. */ dm_table_presuspend_targets(map); /* * Flush I/O to the device. * Any I/O submitted after lock_fs() may not be flushed. * noflush takes precedence over do_lockfs. * (lock_fs() flushes I/Os and waits for them to complete.) */ if (!noflush && do_lockfs) { r = lock_fs(md); if (r) { dm_table_presuspend_undo_targets(map); return r; } } /* * Here we must make sure that no processes are submitting requests * to target drivers i.e. no one may be executing * dm_split_and_process_bio from dm_submit_bio. * * To get all processes out of dm_split_and_process_bio in dm_submit_bio, * we take the write lock. To prevent any process from reentering * dm_split_and_process_bio from dm_submit_bio and quiesce the thread * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call * flush_workqueue(md->wq). */ set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); if (map) synchronize_srcu(&md->io_barrier); /* * Stop md->queue before flushing md->wq in case request-based * dm defers requests to md->wq from md->queue. */ if (map && dm_request_based(md)) { dm_stop_queue(md->queue); set_bit(DMF_QUEUE_STOPPED, &md->flags); } flush_workqueue(md->wq); /* * At this point no more requests are entering target request routines. * We call dm_wait_for_completion to wait for all existing requests * to finish. */ if (map) r = dm_wait_for_completion(md, task_state); if (!r) set_bit(dmf_suspended_flag, &md->flags); if (map) synchronize_srcu(&md->io_barrier); /* were we interrupted ? */ if (r < 0) { dm_queue_flush(md); if (test_and_clear_bit(DMF_QUEUE_STOPPED, &md->flags)) dm_start_queue(md->queue); unlock_fs(md); dm_table_presuspend_undo_targets(map); /* pushback list is already flushed, so skip flush */ } return r; } /* * We need to be able to change a mapping table under a mounted * filesystem. For example we might want to move some data in * the background. Before the table can be swapped with * dm_bind_table, dm_suspend must be called to flush any in * flight bios and ensure that any further io gets deferred. */ /* * Suspend mechanism in request-based dm. * * 1. Flush all I/Os by lock_fs() if needed. * 2. Stop dispatching any I/O by stopping the request_queue. * 3. Wait for all in-flight I/Os to be completed or requeued. * * To abort suspend, start the request_queue. */ int dm_suspend(struct mapped_device *md, unsigned int suspend_flags) { struct dm_table *map = NULL; int r = 0; retry: mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); if (dm_suspended_md(md)) { r = -EINVAL; goto out_unlock; } if (dm_suspended_internally_md(md)) { /* already internally suspended, wait for internal resume */ mutex_unlock(&md->suspend_lock); r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); if (r) return r; goto retry; } map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); if (!map) { /* avoid deadlock with fs/namespace.c:do_mount() */ suspend_flags &= ~DM_SUSPEND_LOCKFS_FLAG; } r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED); if (r) goto out_unlock; set_bit(DMF_POST_SUSPENDING, &md->flags); dm_table_postsuspend_targets(map); clear_bit(DMF_POST_SUSPENDING, &md->flags); out_unlock: mutex_unlock(&md->suspend_lock); return r; } static int __dm_resume(struct mapped_device *md, struct dm_table *map) { if (map) { int r = dm_table_resume_targets(map); if (r) return r; } dm_queue_flush(md); /* * Flushing deferred I/Os must be done after targets are resumed * so that mapping of targets can work correctly. * Request-based dm is queueing the deferred I/Os in its request_queue. */ if (test_and_clear_bit(DMF_QUEUE_STOPPED, &md->flags)) dm_start_queue(md->queue); unlock_fs(md); return 0; } int dm_resume(struct mapped_device *md) { int r; struct dm_table *map = NULL; retry: r = -EINVAL; mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING); if (!dm_suspended_md(md)) goto out; if (dm_suspended_internally_md(md)) { /* already internally suspended, wait for internal resume */ mutex_unlock(&md->suspend_lock); r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE); if (r) return r; goto retry; } map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); if (!map || !dm_table_get_size(map)) goto out; r = __dm_resume(md, map); if (r) goto out; clear_bit(DMF_SUSPENDED, &md->flags); out: mutex_unlock(&md->suspend_lock); return r; } /* * Internal suspend/resume works like userspace-driven suspend. It waits * until all bios finish and prevents issuing new bios to the target drivers. * It may be used only from the kernel. */ static void __dm_internal_suspend(struct mapped_device *md, unsigned int suspend_flags) { struct dm_table *map = NULL; lockdep_assert_held(&md->suspend_lock); if (md->internal_suspend_count++) return; /* nested internal suspend */ if (dm_suspended_md(md)) { set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); return; /* nest suspend */ } map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); /* * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend * would require changing .presuspend to return an error -- avoid this * until there is a need for more elaborate variants of internal suspend. */ (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE, DMF_SUSPENDED_INTERNALLY); set_bit(DMF_POST_SUSPENDING, &md->flags); dm_table_postsuspend_targets(map); clear_bit(DMF_POST_SUSPENDING, &md->flags); } static void __dm_internal_resume(struct mapped_device *md) { int r; struct dm_table *map; BUG_ON(!md->internal_suspend_count); if (--md->internal_suspend_count) return; /* resume from nested internal suspend */ if (dm_suspended_md(md)) goto done; /* resume from nested suspend */ map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock)); r = __dm_resume(md, map); if (r) { /* * If a preresume method of some target failed, we are in a * tricky situation. We can't return an error to the caller. We * can't fake success because then the "resume" and * "postsuspend" methods would not be paired correctly, and it * would break various targets, for example it would cause list * corruption in the "origin" target. * * So, we fake normal suspend here, to make sure that the * "resume" and "postsuspend" methods will be paired correctly. */ DMERR("Preresume method failed: %d", r); set_bit(DMF_SUSPENDED, &md->flags); } done: clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); smp_mb__after_atomic(); wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY); } void dm_internal_suspend_noflush(struct mapped_device *md) { mutex_lock(&md->suspend_lock); __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG); mutex_unlock(&md->suspend_lock); } EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush); void dm_internal_resume(struct mapped_device *md) { mutex_lock(&md->suspend_lock); __dm_internal_resume(md); mutex_unlock(&md->suspend_lock); } EXPORT_SYMBOL_GPL(dm_internal_resume); /* * Fast variants of internal suspend/resume hold md->suspend_lock, * which prevents interaction with userspace-driven suspend. */ void dm_internal_suspend_fast(struct mapped_device *md) { mutex_lock(&md->suspend_lock); if (dm_suspended_md(md) || dm_suspended_internally_md(md)) return; set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags); synchronize_srcu(&md->io_barrier); flush_workqueue(md->wq); dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE); } EXPORT_SYMBOL_GPL(dm_internal_suspend_fast); void dm_internal_resume_fast(struct mapped_device *md) { if (dm_suspended_md(md) || dm_suspended_internally_md(md)) goto done; dm_queue_flush(md); done: mutex_unlock(&md->suspend_lock); } EXPORT_SYMBOL_GPL(dm_internal_resume_fast); /* *--------------------------------------------------------------- * Event notification. *--------------------------------------------------------------- */ int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action, unsigned int cookie, bool need_resize_uevent) { int r; unsigned int noio_flag; char udev_cookie[DM_COOKIE_LENGTH]; char *envp[3] = { NULL, NULL, NULL }; char **envpp = envp; if (cookie) { snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u", DM_COOKIE_ENV_VAR_NAME, cookie); *envpp++ = udev_cookie; } if (need_resize_uevent) { *envpp++ = "RESIZE=1"; } noio_flag = memalloc_noio_save(); r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj, action, envp); memalloc_noio_restore(noio_flag); return r; } uint32_t dm_next_uevent_seq(struct mapped_device *md) { return atomic_add_return(1, &md->uevent_seq); } uint32_t dm_get_event_nr(struct mapped_device *md) { return atomic_read(&md->event_nr); } int dm_wait_event(struct mapped_device *md, int event_nr) { return wait_event_interruptible(md->eventq, (event_nr != atomic_read(&md->event_nr))); } void dm_uevent_add(struct mapped_device *md, struct list_head *elist) { unsigned long flags; spin_lock_irqsave(&md->uevent_lock, flags); list_add(elist, &md->uevent_list); spin_unlock_irqrestore(&md->uevent_lock, flags); } /* * The gendisk is only valid as long as you have a reference * count on 'md'. */ struct gendisk *dm_disk(struct mapped_device *md) { return md->disk; } EXPORT_SYMBOL_GPL(dm_disk); struct kobject *dm_kobject(struct mapped_device *md) { return &md->kobj_holder.kobj; } struct mapped_device *dm_get_from_kobject(struct kobject *kobj) { struct mapped_device *md; md = container_of(kobj, struct mapped_device, kobj_holder.kobj); spin_lock(&_minor_lock); if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) { md = NULL; goto out; } dm_get(md); out: spin_unlock(&_minor_lock); return md; } int dm_suspended_md(struct mapped_device *md) { return test_bit(DMF_SUSPENDED, &md->flags); } static int dm_post_suspending_md(struct mapped_device *md) { return test_bit(DMF_POST_SUSPENDING, &md->flags); } int dm_suspended_internally_md(struct mapped_device *md) { return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags); } int dm_test_deferred_remove_flag(struct mapped_device *md) { return test_bit(DMF_DEFERRED_REMOVE, &md->flags); } int dm_suspended(struct dm_target *ti) { return dm_suspended_md(ti->table->md); } EXPORT_SYMBOL_GPL(dm_suspended); int dm_post_suspending(struct dm_target *ti) { return dm_post_suspending_md(ti->table->md); } EXPORT_SYMBOL_GPL(dm_post_suspending); int dm_noflush_suspending(struct dm_target *ti) { return __noflush_suspending(ti->table->md); } EXPORT_SYMBOL_GPL(dm_noflush_suspending); void dm_free_md_mempools(struct dm_md_mempools *pools) { if (!pools) return; bioset_exit(&pools->bs); bioset_exit(&pools->io_bs); kfree(pools); } struct dm_blkdev_id { u8 *id; enum blk_unique_id type; }; static int __dm_get_unique_id(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { struct dm_blkdev_id *dm_id = data; const struct block_device_operations *fops = dev->bdev->bd_disk->fops; if (!fops->get_unique_id) return 0; return fops->get_unique_id(dev->bdev->bd_disk, dm_id->id, dm_id->type); } /* * Allow access to get_unique_id() for the first device returning a * non-zero result. Reasonable use expects all devices to have the * same unique id. */ static int dm_blk_get_unique_id(struct gendisk *disk, u8 *id, enum blk_unique_id type) { struct mapped_device *md = disk->private_data; struct dm_table *table; struct dm_target *ti; int ret = 0, srcu_idx; struct dm_blkdev_id dm_id = { .id = id, .type = type, }; table = dm_get_live_table(md, &srcu_idx); if (!table || !dm_table_get_size(table)) goto out; /* We only support devices that have a single target */ if (table->num_targets != 1) goto out; ti = dm_table_get_target(table, 0); if (!ti->type->iterate_devices) goto out; ret = ti->type->iterate_devices(ti, __dm_get_unique_id, &dm_id); out: dm_put_live_table(md, srcu_idx); return ret; } struct dm_pr { u64 old_key; u64 new_key; u32 flags; bool abort; bool fail_early; int ret; enum pr_type type; struct pr_keys *read_keys; struct pr_held_reservation *rsv; }; static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn, struct dm_pr *pr) { struct mapped_device *md = bdev->bd_disk->private_data; struct dm_table *table; struct dm_target *ti; int ret = -ENOTTY, srcu_idx; table = dm_get_live_table(md, &srcu_idx); if (!table || !dm_table_get_size(table)) goto out; /* We only support devices that have a single target */ if (table->num_targets != 1) goto out; ti = dm_table_get_target(table, 0); if (dm_suspended_md(md)) { ret = -EAGAIN; goto out; } ret = -EINVAL; if (!ti->type->iterate_devices) goto out; ti->type->iterate_devices(ti, fn, pr); ret = 0; out: dm_put_live_table(md, srcu_idx); return ret; } /* * For register / unregister we need to manually call out to every path. */ static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { struct dm_pr *pr = data; const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; int ret; if (!ops || !ops->pr_register) { pr->ret = -EOPNOTSUPP; return -1; } ret = ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags); if (!ret) return 0; if (!pr->ret) pr->ret = ret; if (pr->fail_early) return -1; return 0; } static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key, u32 flags) { struct dm_pr pr = { .old_key = old_key, .new_key = new_key, .flags = flags, .fail_early = true, .ret = 0, }; int ret; ret = dm_call_pr(bdev, __dm_pr_register, &pr); if (ret) { /* Didn't even get to register a path */ return ret; } if (!pr.ret) return 0; ret = pr.ret; if (!new_key) return ret; /* unregister all paths if we failed to register any path */ pr.old_key = new_key; pr.new_key = 0; pr.flags = 0; pr.fail_early = false; (void) dm_call_pr(bdev, __dm_pr_register, &pr); return ret; } static int __dm_pr_reserve(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { struct dm_pr *pr = data; const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; if (!ops || !ops->pr_reserve) { pr->ret = -EOPNOTSUPP; return -1; } pr->ret = ops->pr_reserve(dev->bdev, pr->old_key, pr->type, pr->flags); if (!pr->ret) return -1; return 0; } static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type, u32 flags) { struct dm_pr pr = { .old_key = key, .flags = flags, .type = type, .fail_early = false, .ret = 0, }; int ret; ret = dm_call_pr(bdev, __dm_pr_reserve, &pr); if (ret) return ret; return pr.ret; } /* * If there is a non-All Registrants type of reservation, the release must be * sent down the holding path. For the cases where there is no reservation or * the path is not the holder the device will also return success, so we must * try each path to make sure we got the correct path. */ static int __dm_pr_release(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { struct dm_pr *pr = data; const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; if (!ops || !ops->pr_release) { pr->ret = -EOPNOTSUPP; return -1; } pr->ret = ops->pr_release(dev->bdev, pr->old_key, pr->type); if (pr->ret) return -1; return 0; } static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type) { struct dm_pr pr = { .old_key = key, .type = type, .fail_early = false, }; int ret; ret = dm_call_pr(bdev, __dm_pr_release, &pr); if (ret) return ret; return pr.ret; } static int __dm_pr_preempt(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { struct dm_pr *pr = data; const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; if (!ops || !ops->pr_preempt) { pr->ret = -EOPNOTSUPP; return -1; } pr->ret = ops->pr_preempt(dev->bdev, pr->old_key, pr->new_key, pr->type, pr->abort); if (!pr->ret) return -1; return 0; } static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key, enum pr_type type, bool abort) { struct dm_pr pr = { .new_key = new_key, .old_key = old_key, .type = type, .fail_early = false, }; int ret; ret = dm_call_pr(bdev, __dm_pr_preempt, &pr); if (ret) return ret; return pr.ret; } static int dm_pr_clear(struct block_device *bdev, u64 key) { struct mapped_device *md = bdev->bd_disk->private_data; const struct pr_ops *ops; int r, srcu_idx; bool forward = true; /* Not a real ioctl, but targets must not interpret non-DM ioctls */ r = dm_prepare_ioctl(md, &srcu_idx, &bdev, 0, 0, &forward); if (r < 0) goto out; WARN_ON_ONCE(!forward); ops = bdev->bd_disk->fops->pr_ops; if (ops && ops->pr_clear) r = ops->pr_clear(bdev, key); else r = -EOPNOTSUPP; out: dm_unprepare_ioctl(md, srcu_idx); return r; } static int __dm_pr_read_keys(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { struct dm_pr *pr = data; const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; if (!ops || !ops->pr_read_keys) { pr->ret = -EOPNOTSUPP; return -1; } pr->ret = ops->pr_read_keys(dev->bdev, pr->read_keys); if (!pr->ret) return -1; return 0; } static int dm_pr_read_keys(struct block_device *bdev, struct pr_keys *keys) { struct dm_pr pr = { .read_keys = keys, }; int ret; ret = dm_call_pr(bdev, __dm_pr_read_keys, &pr); if (ret) return ret; return pr.ret; } static int __dm_pr_read_reservation(struct dm_target *ti, struct dm_dev *dev, sector_t start, sector_t len, void *data) { struct dm_pr *pr = data; const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops; if (!ops || !ops->pr_read_reservation) { pr->ret = -EOPNOTSUPP; return -1; } pr->ret = ops->pr_read_reservation(dev->bdev, pr->rsv); if (!pr->ret) return -1; return 0; } static int dm_pr_read_reservation(struct block_device *bdev, struct pr_held_reservation *rsv) { struct dm_pr pr = { .rsv = rsv, }; int ret; ret = dm_call_pr(bdev, __dm_pr_read_reservation, &pr); if (ret) return ret; return pr.ret; } static const struct pr_ops dm_pr_ops = { .pr_register = dm_pr_register, .pr_reserve = dm_pr_reserve, .pr_release = dm_pr_release, .pr_preempt = dm_pr_preempt, .pr_clear = dm_pr_clear, .pr_read_keys = dm_pr_read_keys, .pr_read_reservation = dm_pr_read_reservation, }; static const struct block_device_operations dm_blk_dops = { .submit_bio = dm_submit_bio, .poll_bio = dm_poll_bio, .open = dm_blk_open, .release = dm_blk_close, .ioctl = dm_blk_ioctl, .getgeo = dm_blk_getgeo, .report_zones = dm_blk_report_zones, .get_unique_id = dm_blk_get_unique_id, .pr_ops = &dm_pr_ops, .owner = THIS_MODULE }; static const struct block_device_operations dm_rq_blk_dops = { .open = dm_blk_open, .release = dm_blk_close, .ioctl = dm_blk_ioctl, .getgeo = dm_blk_getgeo, .get_unique_id = dm_blk_get_unique_id, .pr_ops = &dm_pr_ops, .owner = THIS_MODULE }; static const struct dax_operations dm_dax_ops = { .direct_access = dm_dax_direct_access, .zero_page_range = dm_dax_zero_page_range, .recovery_write = dm_dax_recovery_write, }; /* * module hooks */ module_init(dm_init); module_exit(dm_exit); module_param(major, uint, 0); MODULE_PARM_DESC(major, "The major number of the device mapper"); module_param(reserved_bio_based_ios, uint, 0644); MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools"); module_param(dm_numa_node, int, 0644); MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations"); module_param(swap_bios, int, 0644); MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs"); MODULE_DESCRIPTION(DM_NAME " driver"); MODULE_AUTHOR("Joe Thornber <dm-devel@lists.linux.dev>"); MODULE_LICENSE("GPL"); |
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 | // SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2013 Trond Myklebust <Trond.Myklebust@netapp.com> */ #include <linux/nfs_fs.h> #include <linux/namei.h> #include "internal.h" #define CREATE_TRACE_POINTS #include "nfstrace.h" EXPORT_TRACEPOINT_SYMBOL_GPL(nfs_fsync_enter); EXPORT_TRACEPOINT_SYMBOL_GPL(nfs_fsync_exit); EXPORT_TRACEPOINT_SYMBOL_GPL(nfs_xdr_status); EXPORT_TRACEPOINT_SYMBOL_GPL(nfs_xdr_bad_filehandle); |
| 1 1 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 | // SPDX-License-Identifier: GPL-2.0 /* * Renesas Electronics uPD78F0730 USB to serial converter driver * * Copyright (C) 2014,2016 Maksim Salau <maksim.salau@gmail.com> * * Protocol of the adaptor is described in the application note U19660EJ1V0AN00 * μPD78F0730 8-bit Single-Chip Microcontroller * USB-to-Serial Conversion Software * <https://www.renesas.com/en-eu/doc/DocumentServer/026/U19660EJ1V0AN00.pdf> * * The adaptor functionality is limited to the following: * - data bits: 7 or 8 * - stop bits: 1 or 2 * - parity: even, odd or none * - flow control: none * - baud rates: 0, 2400, 4800, 9600, 19200, 38400, 57600, 115200, 153600 * - signals: DTR, RTS and BREAK */ #include <linux/module.h> #include <linux/slab.h> #include <linux/tty.h> #include <linux/usb.h> #include <linux/usb/serial.h> #define DRIVER_DESC "Renesas uPD78F0730 USB to serial converter driver" #define DRIVER_AUTHOR "Maksim Salau <maksim.salau@gmail.com>" static const struct usb_device_id id_table[] = { { USB_DEVICE(0x0409, 0x0063) }, /* V850ESJX3-STICK */ { USB_DEVICE(0x045B, 0x0212) }, /* YRPBRL78G13, YRPBRL78G14 */ { USB_DEVICE(0x064B, 0x7825) }, /* Analog Devices EVAL-ADXL362Z-DB */ {} }; MODULE_DEVICE_TABLE(usb, id_table); /* * Each adaptor is associated with a private structure, that holds the current * state of control signals (DTR, RTS and BREAK). */ struct upd78f0730_port_private { struct mutex lock; /* mutex to protect line_signals */ u8 line_signals; }; /* Op-codes of control commands */ #define UPD78F0730_CMD_LINE_CONTROL 0x00 #define UPD78F0730_CMD_SET_DTR_RTS 0x01 #define UPD78F0730_CMD_SET_XON_XOFF_CHR 0x02 #define UPD78F0730_CMD_OPEN_CLOSE 0x03 #define UPD78F0730_CMD_SET_ERR_CHR 0x04 /* Data sizes in UPD78F0730_CMD_LINE_CONTROL command */ #define UPD78F0730_DATA_SIZE_7_BITS 0x00 #define UPD78F0730_DATA_SIZE_8_BITS 0x01 #define UPD78F0730_DATA_SIZE_MASK 0x01 /* Stop-bit modes in UPD78F0730_CMD_LINE_CONTROL command */ #define UPD78F0730_STOP_BIT_1_BIT 0x00 #define UPD78F0730_STOP_BIT_2_BIT 0x02 #define UPD78F0730_STOP_BIT_MASK 0x02 /* Parity modes in UPD78F0730_CMD_LINE_CONTROL command */ #define UPD78F0730_PARITY_NONE 0x00 #define UPD78F0730_PARITY_EVEN 0x04 #define UPD78F0730_PARITY_ODD 0x08 #define UPD78F0730_PARITY_MASK 0x0C /* Flow control modes in UPD78F0730_CMD_LINE_CONTROL command */ #define UPD78F0730_FLOW_CONTROL_NONE 0x00 #define UPD78F0730_FLOW_CONTROL_HW 0x10 #define UPD78F0730_FLOW_CONTROL_SW 0x20 #define UPD78F0730_FLOW_CONTROL_MASK 0x30 /* Control signal bits in UPD78F0730_CMD_SET_DTR_RTS command */ #define UPD78F0730_RTS 0x01 #define UPD78F0730_DTR 0x02 #define UPD78F0730_BREAK 0x04 /* Port modes in UPD78F0730_CMD_OPEN_CLOSE command */ #define UPD78F0730_PORT_CLOSE 0x00 #define UPD78F0730_PORT_OPEN 0x01 /* Error character substitution modes in UPD78F0730_CMD_SET_ERR_CHR command */ #define UPD78F0730_ERR_CHR_DISABLED 0x00 #define UPD78F0730_ERR_CHR_ENABLED 0x01 /* * Declaration of command structures */ /* UPD78F0730_CMD_LINE_CONTROL command */ struct upd78f0730_line_control { u8 opcode; __le32 baud_rate; u8 params; } __packed; /* UPD78F0730_CMD_SET_DTR_RTS command */ struct upd78f0730_set_dtr_rts { u8 opcode; u8 params; }; /* UPD78F0730_CMD_SET_XON_OFF_CHR command */ struct upd78f0730_set_xon_xoff_chr { u8 opcode; u8 xon; u8 xoff; }; /* UPD78F0730_CMD_OPEN_CLOSE command */ struct upd78f0730_open_close { u8 opcode; u8 state; }; /* UPD78F0730_CMD_SET_ERR_CHR command */ struct upd78f0730_set_err_chr { u8 opcode; u8 state; u8 err_char; }; static int upd78f0730_send_ctl(struct usb_serial_port *port, const void *data, int size) { struct usb_device *usbdev = port->serial->dev; void *buf; int res; if (size <= 0 || !data) return -EINVAL; buf = kmemdup(data, size, GFP_KERNEL); if (!buf) return -ENOMEM; res = usb_control_msg(usbdev, usb_sndctrlpipe(usbdev, 0), 0x00, USB_TYPE_VENDOR | USB_RECIP_DEVICE | USB_DIR_OUT, 0x0000, 0x0000, buf, size, USB_CTRL_SET_TIMEOUT); kfree(buf); if (res < 0) { struct device *dev = &port->dev; dev_err(dev, "failed to send control request %02x: %d\n", *(u8 *)data, res); return res; } return 0; } static int upd78f0730_port_probe(struct usb_serial_port *port) { struct upd78f0730_port_private *private; private = kzalloc_obj(*private); if (!private) return -ENOMEM; mutex_init(&private->lock); usb_set_serial_port_data(port, private); return 0; } static void upd78f0730_port_remove(struct usb_serial_port *port) { struct upd78f0730_port_private *private; private = usb_get_serial_port_data(port); mutex_destroy(&private->lock); kfree(private); } static int upd78f0730_tiocmget(struct tty_struct *tty) { struct upd78f0730_port_private *private; struct usb_serial_port *port = tty->driver_data; int signals; int res; private = usb_get_serial_port_data(port); mutex_lock(&private->lock); signals = private->line_signals; mutex_unlock(&private->lock); res = ((signals & UPD78F0730_DTR) ? TIOCM_DTR : 0) | ((signals & UPD78F0730_RTS) ? TIOCM_RTS : 0); dev_dbg(&port->dev, "%s - res = %x\n", __func__, res); return res; } static int upd78f0730_tiocmset(struct tty_struct *tty, unsigned int set, unsigned int clear) { struct usb_serial_port *port = tty->driver_data; struct upd78f0730_port_private *private; struct upd78f0730_set_dtr_rts request; struct device *dev = &port->dev; int res; private = usb_get_serial_port_data(port); mutex_lock(&private->lock); if (set & TIOCM_DTR) { private->line_signals |= UPD78F0730_DTR; dev_dbg(dev, "%s - set DTR\n", __func__); } if (set & TIOCM_RTS) { private->line_signals |= UPD78F0730_RTS; dev_dbg(dev, "%s - set RTS\n", __func__); } if (clear & TIOCM_DTR) { private->line_signals &= ~UPD78F0730_DTR; dev_dbg(dev, "%s - clear DTR\n", __func__); } if (clear & TIOCM_RTS) { private->line_signals &= ~UPD78F0730_RTS; dev_dbg(dev, "%s - clear RTS\n", __func__); } request.opcode = UPD78F0730_CMD_SET_DTR_RTS; request.params = private->line_signals; res = upd78f0730_send_ctl(port, &request, sizeof(request)); mutex_unlock(&private->lock); return res; } static int upd78f0730_break_ctl(struct tty_struct *tty, int break_state) { struct upd78f0730_port_private *private; struct usb_serial_port *port = tty->driver_data; struct upd78f0730_set_dtr_rts request; struct device *dev = &port->dev; int res; private = usb_get_serial_port_data(port); mutex_lock(&private->lock); if (break_state) { private->line_signals |= UPD78F0730_BREAK; dev_dbg(dev, "%s - set BREAK\n", __func__); } else { private->line_signals &= ~UPD78F0730_BREAK; dev_dbg(dev, "%s - clear BREAK\n", __func__); } request.opcode = UPD78F0730_CMD_SET_DTR_RTS; request.params = private->line_signals; res = upd78f0730_send_ctl(port, &request, sizeof(request)); mutex_unlock(&private->lock); return res; } static void upd78f0730_dtr_rts(struct usb_serial_port *port, int on) { struct tty_struct *tty = port->port.tty; unsigned int set = 0; unsigned int clear = 0; if (on) set = TIOCM_DTR | TIOCM_RTS; else clear = TIOCM_DTR | TIOCM_RTS; upd78f0730_tiocmset(tty, set, clear); } static speed_t upd78f0730_get_baud_rate(struct tty_struct *tty) { const speed_t baud_rate = tty_get_baud_rate(tty); static const speed_t supported[] = { 0, 2400, 4800, 9600, 19200, 38400, 57600, 115200, 153600 }; int i; for (i = ARRAY_SIZE(supported) - 1; i >= 0; i--) { if (baud_rate == supported[i]) return baud_rate; } /* If the baud rate is not supported, switch to the default one */ tty_encode_baud_rate(tty, 9600, 9600); return tty_get_baud_rate(tty); } static void upd78f0730_set_termios(struct tty_struct *tty, struct usb_serial_port *port, const struct ktermios *old_termios) { struct device *dev = &port->dev; struct upd78f0730_line_control request; speed_t baud_rate; if (old_termios && !tty_termios_hw_change(&tty->termios, old_termios)) return; if (C_BAUD(tty) == B0) upd78f0730_dtr_rts(port, 0); else if (old_termios && (old_termios->c_cflag & CBAUD) == B0) upd78f0730_dtr_rts(port, 1); baud_rate = upd78f0730_get_baud_rate(tty); request.opcode = UPD78F0730_CMD_LINE_CONTROL; request.baud_rate = cpu_to_le32(baud_rate); request.params = 0; dev_dbg(dev, "%s - baud rate = %d\n", __func__, baud_rate); switch (C_CSIZE(tty)) { case CS7: request.params |= UPD78F0730_DATA_SIZE_7_BITS; dev_dbg(dev, "%s - 7 data bits\n", __func__); break; default: tty->termios.c_cflag &= ~CSIZE; tty->termios.c_cflag |= CS8; dev_warn(dev, "data size is not supported, using 8 bits\n"); fallthrough; case CS8: request.params |= UPD78F0730_DATA_SIZE_8_BITS; dev_dbg(dev, "%s - 8 data bits\n", __func__); break; } if (C_PARENB(tty)) { if (C_PARODD(tty)) { request.params |= UPD78F0730_PARITY_ODD; dev_dbg(dev, "%s - odd parity\n", __func__); } else { request.params |= UPD78F0730_PARITY_EVEN; dev_dbg(dev, "%s - even parity\n", __func__); } if (C_CMSPAR(tty)) { tty->termios.c_cflag &= ~CMSPAR; dev_warn(dev, "MARK/SPACE parity is not supported\n"); } } else { request.params |= UPD78F0730_PARITY_NONE; dev_dbg(dev, "%s - no parity\n", __func__); } if (C_CSTOPB(tty)) { request.params |= UPD78F0730_STOP_BIT_2_BIT; dev_dbg(dev, "%s - 2 stop bits\n", __func__); } else { request.params |= UPD78F0730_STOP_BIT_1_BIT; dev_dbg(dev, "%s - 1 stop bit\n", __func__); } if (C_CRTSCTS(tty)) { tty->termios.c_cflag &= ~CRTSCTS; dev_warn(dev, "RTSCTS flow control is not supported\n"); } if (I_IXOFF(tty) || I_IXON(tty)) { tty->termios.c_iflag &= ~(IXOFF | IXON); dev_warn(dev, "XON/XOFF flow control is not supported\n"); } request.params |= UPD78F0730_FLOW_CONTROL_NONE; dev_dbg(dev, "%s - no flow control\n", __func__); upd78f0730_send_ctl(port, &request, sizeof(request)); } static int upd78f0730_open(struct tty_struct *tty, struct usb_serial_port *port) { static const struct upd78f0730_open_close request = { .opcode = UPD78F0730_CMD_OPEN_CLOSE, .state = UPD78F0730_PORT_OPEN }; int res; res = upd78f0730_send_ctl(port, &request, sizeof(request)); if (res) return res; if (tty) upd78f0730_set_termios(tty, port, NULL); return usb_serial_generic_open(tty, port); } static void upd78f0730_close(struct usb_serial_port *port) { static const struct upd78f0730_open_close request = { .opcode = UPD78F0730_CMD_OPEN_CLOSE, .state = UPD78F0730_PORT_CLOSE }; usb_serial_generic_close(port); upd78f0730_send_ctl(port, &request, sizeof(request)); } static struct usb_serial_driver upd78f0730_device = { .driver = { .name = "upd78f0730", }, .id_table = id_table, .num_ports = 1, .port_probe = upd78f0730_port_probe, .port_remove = upd78f0730_port_remove, .open = upd78f0730_open, .close = upd78f0730_close, .set_termios = upd78f0730_set_termios, .tiocmget = upd78f0730_tiocmget, .tiocmset = upd78f0730_tiocmset, .dtr_rts = upd78f0730_dtr_rts, .break_ctl = upd78f0730_break_ctl, }; static struct usb_serial_driver * const serial_drivers[] = { &upd78f0730_device, NULL }; module_usb_serial_driver(serial_drivers, id_table); MODULE_DESCRIPTION(DRIVER_DESC); MODULE_AUTHOR(DRIVER_AUTHOR); MODULE_LICENSE("GPL v2"); |
| 13 11 11 10 10 10 1 9 9 3 8 9 9 9 9 9 8 9 9 9 1 1 10 10 9 10 5 5 4 3 2 2 1 7 2 7 7 7 7 7 2 2 1 1 2 7 7 7 7 7 6 7 8 8 8 8 8 8 7 8 8 14 14 14 14 14 14 14 14 14 14 13 10 7 14 14 3 2 12 12 5 12 14 14 1 1 1 1 1 1 1 1 13 4 13 13 12 10 12 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 15 14 13 1 15 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 | // SPDX-License-Identifier: GPL-2.0 /* FUSE: Filesystem in Userspace Copyright (C) 2001-2018 Miklos Szeredi <miklos@szeredi.hu> */ #include "fuse_i.h" #include <linux/iversion.h> #include <linux/posix_acl.h> #include <linux/pagemap.h> #include <linux/highmem.h> #include <linux/vmalloc.h> static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx) { struct fuse_conn *fc = get_fuse_conn(dir); struct fuse_inode *fi = get_fuse_inode(dir); if (!fc->do_readdirplus) return false; if (!fc->readdirplus_auto) return true; if (test_and_clear_bit(FUSE_I_ADVISE_RDPLUS, &fi->state)) return true; if (ctx->pos == 0) return true; return false; } static void fuse_add_dirent_to_cache(struct file *file, struct fuse_dirent *dirent, loff_t pos) { struct fuse_inode *fi = get_fuse_inode(file_inode(file)); size_t reclen = FUSE_DIRENT_SIZE(dirent); pgoff_t index; struct page *page; loff_t size; u64 version; unsigned int offset; void *addr; /* Dirent doesn't fit in readdir cache page? Skip caching. */ if (reclen > PAGE_SIZE) return; spin_lock(&fi->rdc.lock); /* * Is cache already completed? Or this entry does not go at the end of * cache? */ if (fi->rdc.cached || pos != fi->rdc.pos) { spin_unlock(&fi->rdc.lock); return; } version = fi->rdc.version; size = fi->rdc.size; offset = offset_in_page(size); index = size >> PAGE_SHIFT; /* Dirent doesn't fit in current page? Jump to next page. */ if (offset + reclen > PAGE_SIZE) { index++; offset = 0; } spin_unlock(&fi->rdc.lock); if (offset) { page = find_lock_page(file->f_mapping, index); } else { page = find_or_create_page(file->f_mapping, index, mapping_gfp_mask(file->f_mapping)); } if (!page) return; spin_lock(&fi->rdc.lock); /* Raced with another readdir */ if (fi->rdc.version != version || fi->rdc.size != size || WARN_ON(fi->rdc.pos != pos)) goto unlock; addr = kmap_local_page(page); if (!offset) { clear_page(addr); SetPageUptodate(page); } memcpy(addr + offset, dirent, reclen); kunmap_local(addr); fi->rdc.size = (index << PAGE_SHIFT) + offset + reclen; fi->rdc.pos = dirent->off; unlock: spin_unlock(&fi->rdc.lock); unlock_page(page); put_page(page); } static void fuse_readdir_cache_end(struct file *file, loff_t pos) { struct fuse_inode *fi = get_fuse_inode(file_inode(file)); loff_t end; spin_lock(&fi->rdc.lock); /* does cache end position match current position? */ if (fi->rdc.pos != pos) { spin_unlock(&fi->rdc.lock); return; } fi->rdc.cached = true; end = ALIGN(fi->rdc.size, PAGE_SIZE); spin_unlock(&fi->rdc.lock); /* truncate unused tail of cache */ truncate_inode_pages(file->f_mapping, end); } static bool fuse_emit(struct file *file, struct dir_context *ctx, struct fuse_dirent *dirent) { struct fuse_file *ff = file->private_data; if (ff->open_flags & FOPEN_CACHE_DIR) fuse_add_dirent_to_cache(file, dirent, ctx->pos); return dir_emit(ctx, dirent->name, dirent->namelen, dirent->ino, dirent->type | FILLDIR_FLAG_NOINTR); } static int parse_dirfile(char *buf, size_t nbytes, struct file *file, struct dir_context *ctx) { while (nbytes >= FUSE_NAME_OFFSET) { struct fuse_dirent *dirent = (struct fuse_dirent *) buf; size_t reclen = FUSE_DIRENT_SIZE(dirent); if (!dirent->namelen || dirent->namelen > FUSE_NAME_MAX) return -EIO; if (reclen > nbytes) break; if (memchr(dirent->name, '/', dirent->namelen) != NULL) return -EIO; if (!fuse_emit(file, ctx, dirent)) break; buf += reclen; nbytes -= reclen; ctx->pos = dirent->off; } return 0; } static int fuse_direntplus_link(struct file *file, struct fuse_direntplus *direntplus, u64 attr_version, u64 evict_ctr) { struct fuse_entry_out *o = &direntplus->entry_out; struct fuse_dirent *dirent = &direntplus->dirent; struct dentry *parent = file->f_path.dentry; struct qstr name = QSTR_INIT(dirent->name, dirent->namelen); struct dentry *dentry; struct dentry *alias; struct inode *dir = d_inode(parent); struct fuse_conn *fc; struct inode *inode; int epoch; if (!o->nodeid) { /* * Unlike in the case of fuse_lookup, zero nodeid does not mean * ENOENT. Instead, it only means the userspace filesystem did * not want to return attributes/handle for this entry. * * So do nothing. */ return 0; } if (name.name[0] == '.') { /* * We could potentially refresh the attributes of the directory * and its parent? */ if (name.len == 1) return 0; if (name.name[1] == '.' && name.len == 2) return 0; } if (invalid_nodeid(o->nodeid)) return -EIO; if (fuse_invalid_attr(&o->attr)) return -EIO; fc = get_fuse_conn(dir); epoch = atomic_read(&fc->epoch); name.hash = full_name_hash(parent, name.name, name.len); dentry = d_lookup(parent, &name); if (!dentry) { retry: dentry = d_alloc_parallel(parent, &name); if (IS_ERR(dentry)) return PTR_ERR(dentry); } if (!d_in_lookup(dentry)) { struct fuse_inode *fi; inode = d_inode(dentry); if (inode && get_node_id(inode) != o->nodeid) inode = NULL; if (!inode || fuse_stale_inode(inode, o->generation, &o->attr)) { if (inode) fuse_make_bad(inode); d_invalidate(dentry); dput(dentry); goto retry; } if (fuse_is_bad(inode)) { dput(dentry); return -EIO; } fi = get_fuse_inode(inode); spin_lock(&fi->lock); fi->nlookup++; spin_unlock(&fi->lock); forget_all_cached_acls(inode); fuse_change_attributes(inode, &o->attr, NULL, ATTR_TIMEOUT(o), attr_version); /* * The other branch comes via fuse_iget() * which bumps nlookup inside */ } else { inode = fuse_iget(dir->i_sb, o->nodeid, o->generation, &o->attr, ATTR_TIMEOUT(o), attr_version, evict_ctr); if (!inode) inode = ERR_PTR(-ENOMEM); alias = d_splice_alias(inode, dentry); d_lookup_done(dentry); if (alias) { dput(dentry); dentry = alias; } if (IS_ERR(dentry)) { if (!IS_ERR(inode)) { struct fuse_inode *fi = get_fuse_inode(inode); spin_lock(&fi->lock); fi->nlookup--; spin_unlock(&fi->lock); } return PTR_ERR(dentry); } } if (fc->readdirplus_auto) set_bit(FUSE_I_INIT_RDPLUS, &get_fuse_inode(inode)->state); dentry->d_time = epoch; fuse_change_entry_timeout(dentry, o); dput(dentry); return 0; } static void fuse_force_forget(struct file *file, u64 nodeid) { struct inode *inode = file_inode(file); struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_forget_in inarg; FUSE_ARGS(args); memset(&inarg, 0, sizeof(inarg)); inarg.nlookup = 1; args.opcode = FUSE_FORGET; args.nodeid = nodeid; args.in_numargs = 1; args.in_args[0].size = sizeof(inarg); args.in_args[0].value = &inarg; args.force = true; args.noreply = true; fuse_simple_request(fm, &args); /* ignore errors */ } static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, struct dir_context *ctx, u64 attr_version, u64 evict_ctr) { struct fuse_direntplus *direntplus; struct fuse_dirent *dirent; size_t reclen; int over = 0; int ret; while (nbytes >= FUSE_NAME_OFFSET_DIRENTPLUS) { direntplus = (struct fuse_direntplus *) buf; dirent = &direntplus->dirent; reclen = FUSE_DIRENTPLUS_SIZE(direntplus); if (!dirent->namelen || dirent->namelen > FUSE_NAME_MAX) return -EIO; if (reclen > nbytes) break; if (memchr(dirent->name, '/', dirent->namelen) != NULL) return -EIO; if (!over) { /* We fill entries into dstbuf only as much as it can hold. But we still continue iterating over remaining entries to link them. If not, we need to send a FORGET for each of those which we did not link. */ over = !fuse_emit(file, ctx, dirent); if (!over) ctx->pos = dirent->off; } buf += reclen; nbytes -= reclen; ret = fuse_direntplus_link(file, direntplus, attr_version, evict_ctr); if (ret) fuse_force_forget(file, direntplus->entry_out.nodeid); } return 0; } static struct page **fuse_readdir_alloc_buf(struct fuse_args_pages *ap, size_t *bufsize) { unsigned int i, nr_alloc, nr_pages = DIV_ROUND_UP(*bufsize, PAGE_SIZE); struct page **pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL); if (!pages) return NULL; nr_alloc = alloc_pages_bulk(GFP_KERNEL, nr_pages, pages); if (!nr_alloc) goto free_array; if (nr_alloc < nr_pages) { nr_pages = nr_alloc; *bufsize = (size_t) nr_pages << PAGE_SHIFT; } ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); if (!ap->folios) goto release_pages; for (i = 0; i < nr_pages; i++) { ap->folios[i] = page_folio(pages[i]); ap->descs[i].length = min_t(size_t, *bufsize - (size_t)i * PAGE_SIZE, PAGE_SIZE); } ap->num_folios = nr_pages; ap->args.out_pages = true; return pages; release_pages: release_pages(pages, nr_pages); free_array: kfree(pages); return NULL; } static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx) { int plus; ssize_t res; struct inode *inode = file_inode(file); struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_conn *fc = fm->fc; struct fuse_io_args ia = {}; struct fuse_args_pages *ap = &ia.ap; void *buf; size_t bufsize = clamp((unsigned int) ctx->count, PAGE_SIZE, fc->max_pages << PAGE_SHIFT); u64 attr_version = 0, evict_ctr = 0; bool locked; struct page **pages = fuse_readdir_alloc_buf(ap, &bufsize); if (!pages) return -ENOMEM; plus = fuse_use_readdirplus(inode, ctx); if (plus) { attr_version = fuse_get_attr_version(fm->fc); evict_ctr = fuse_get_evict_ctr(fm->fc); fuse_read_args_fill(&ia, file, ctx->pos, bufsize, FUSE_READDIRPLUS); } else { fuse_read_args_fill(&ia, file, ctx->pos, bufsize, FUSE_READDIR); } locked = fuse_lock_inode(inode); res = fuse_simple_request(fm, &ap->args); fuse_unlock_inode(inode, locked); if (res < 0) goto out; if (!res) { struct fuse_file *ff = file->private_data; if (ff->open_flags & FOPEN_CACHE_DIR) fuse_readdir_cache_end(file, ctx->pos); goto out; } buf = vm_map_ram(pages, ap->num_folios, -1); if (!buf) { res = -ENOMEM; } else { if (plus) res = parse_dirplusfile(buf, res, file, ctx, attr_version, evict_ctr); else res = parse_dirfile(buf, res, file, ctx); vm_unmap_ram(buf, ap->num_folios); } out: kfree(ap->folios); release_pages(pages, ap->num_folios); kfree(pages); fuse_invalidate_atime(inode); return res; } enum fuse_parse_result { FOUND_ERR = -1, FOUND_NONE = 0, FOUND_SOME, FOUND_ALL, }; static enum fuse_parse_result fuse_parse_cache(struct fuse_file *ff, void *addr, unsigned int size, struct dir_context *ctx) { unsigned int offset = offset_in_page(ff->readdir.cache_off); enum fuse_parse_result res = FOUND_NONE; WARN_ON(offset >= size); for (;;) { struct fuse_dirent *dirent = addr + offset; unsigned int nbytes = size - offset; size_t reclen; if (nbytes < FUSE_NAME_OFFSET || !dirent->namelen) break; reclen = FUSE_DIRENT_SIZE(dirent); /* derefs ->namelen */ if (WARN_ON(dirent->namelen > FUSE_NAME_MAX)) return FOUND_ERR; if (WARN_ON(reclen > nbytes)) return FOUND_ERR; if (WARN_ON(memchr(dirent->name, '/', dirent->namelen) != NULL)) return FOUND_ERR; if (ff->readdir.pos == ctx->pos) { res = FOUND_SOME; if (!dir_emit(ctx, dirent->name, dirent->namelen, dirent->ino, dirent->type | FILLDIR_FLAG_NOINTR)) return FOUND_ALL; ctx->pos = dirent->off; } ff->readdir.pos = dirent->off; ff->readdir.cache_off += reclen; offset += reclen; } return res; } static void fuse_rdc_reset(struct inode *inode) { struct fuse_inode *fi = get_fuse_inode(inode); fi->rdc.cached = false; fi->rdc.version++; fi->rdc.size = 0; fi->rdc.pos = 0; fi->rdc.epoch = 0; } #define UNCACHED 1 static int fuse_readdir_cached(struct file *file, struct dir_context *ctx) { struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); enum fuse_parse_result res; pgoff_t index; unsigned int size; struct page *page; void *addr; /* Seeked? If so, reset the cache stream */ if (ff->readdir.pos != ctx->pos) { ff->readdir.pos = 0; ff->readdir.cache_off = 0; } /* * We're just about to start reading into the cache or reading the * cache; both cases require an up-to-date mtime value. */ if (!ctx->pos && fc->auto_inval_data) { int err = fuse_update_attributes(inode, file, STATX_MTIME); if (err) return err; } retry: spin_lock(&fi->rdc.lock); retry_locked: if (!fi->rdc.cached) { /* Starting cache? Set cache mtime. */ if (!ctx->pos && !fi->rdc.size) { fi->rdc.mtime = inode_get_mtime(inode); fi->rdc.iversion = inode_query_iversion(inode); fi->rdc.epoch = atomic_read(&fc->epoch); } spin_unlock(&fi->rdc.lock); return UNCACHED; } /* * When at the beginning of the directory (i.e. just after opendir(3) or * rewinddir(3)), then need to check whether directory contents have * changed, and reset the cache if so. */ if (!ctx->pos) { struct timespec64 mtime = inode_get_mtime(inode); if (inode_peek_iversion(inode) != fi->rdc.iversion || !timespec64_equal(&fi->rdc.mtime, &mtime) || fi->rdc.epoch != atomic_read(&fc->epoch)) { fuse_rdc_reset(inode); goto retry_locked; } } /* * If cache version changed since the last getdents() call, then reset * the cache stream. */ if (ff->readdir.version != fi->rdc.version) { ff->readdir.pos = 0; ff->readdir.cache_off = 0; } /* * If at the beginning of the cache, than reset version to * current. */ if (ff->readdir.pos == 0) ff->readdir.version = fi->rdc.version; WARN_ON(fi->rdc.size < ff->readdir.cache_off); index = ff->readdir.cache_off >> PAGE_SHIFT; if (index == (fi->rdc.size >> PAGE_SHIFT)) size = offset_in_page(fi->rdc.size); else size = PAGE_SIZE; spin_unlock(&fi->rdc.lock); /* EOF? */ if (offset_in_page(ff->readdir.cache_off) == size) return 0; page = find_get_page_flags(file->f_mapping, index, FGP_ACCESSED | FGP_LOCK); /* Page gone missing, then re-added to cache, but not initialized? */ if (page && !PageUptodate(page)) { unlock_page(page); put_page(page); page = NULL; } spin_lock(&fi->rdc.lock); if (!page) { /* * Uh-oh: page gone missing, cache is useless */ if (fi->rdc.version == ff->readdir.version) fuse_rdc_reset(inode); goto retry_locked; } /* Make sure it's still the same version after getting the page. */ if (ff->readdir.version != fi->rdc.version) { spin_unlock(&fi->rdc.lock); unlock_page(page); put_page(page); goto retry; } spin_unlock(&fi->rdc.lock); /* * Contents of the page are now protected against changing by holding * the page lock. */ addr = kmap_local_page(page); res = fuse_parse_cache(ff, addr, size, ctx); kunmap_local(addr); unlock_page(page); put_page(page); if (res == FOUND_ERR) return -EIO; if (res == FOUND_ALL) return 0; if (size == PAGE_SIZE) { /* We hit end of page: skip to next page. */ ff->readdir.cache_off = ALIGN(ff->readdir.cache_off, PAGE_SIZE); goto retry; } /* * End of cache reached. If found position, then we are done, otherwise * need to fall back to uncached, since the position we were looking for * wasn't in the cache. */ return res == FOUND_SOME ? 0 : UNCACHED; } int fuse_readdir(struct file *file, struct dir_context *ctx) { struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(file); int err; if (fuse_is_bad(inode)) return -EIO; err = UNCACHED; if (ff->open_flags & FOPEN_CACHE_DIR) err = fuse_readdir_cached(file, ctx); if (err == UNCACHED) err = fuse_readdir_uncached(file, ctx); return err; } |
| 223 222 222 222 222 222 221 222 10 10 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 | // SPDX-License-Identifier: GPL-2.0 /* * f2fs debugging statistics * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ * Copyright (c) 2012 Linux Foundation * Copyright (c) 2012 Greg Kroah-Hartman <gregkh@linuxfoundation.org> */ #include <linux/fs.h> #include <linux/backing-dev.h> #include <linux/f2fs_fs.h> #include <linux/blkdev.h> #include <linux/debugfs.h> #include <linux/seq_file.h> #include "f2fs.h" #include "node.h" #include "segment.h" #include "gc.h" static LIST_HEAD(f2fs_stat_list); static DEFINE_SPINLOCK(f2fs_stat_lock); #ifdef CONFIG_DEBUG_FS static struct dentry *f2fs_debugfs_root; #endif /* * This function calculates BDF of every segments */ void f2fs_update_sit_info(struct f2fs_sb_info *sbi) { struct f2fs_stat_info *si = F2FS_STAT(sbi); unsigned long long blks_per_sec, hblks_per_sec, total_vblocks; unsigned long long bimodal, dist; unsigned int segno, vblocks; int ndirty = 0; bimodal = 0; total_vblocks = 0; blks_per_sec = CAP_BLKS_PER_SEC(sbi); hblks_per_sec = blks_per_sec / 2; for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) { vblocks = get_valid_blocks(sbi, segno, true); dist = abs(vblocks - hblks_per_sec); bimodal += dist * dist; if (vblocks > 0 && vblocks < blks_per_sec) { total_vblocks += vblocks; ndirty++; } } dist = div_u64(MAIN_SECS(sbi) * hblks_per_sec * hblks_per_sec, 100); si->bimodal = div64_u64(bimodal, dist); if (si->dirty_count) si->avg_vblocks = div_u64(total_vblocks, ndirty); else si->avg_vblocks = 0; } #ifdef CONFIG_DEBUG_FS static void update_multidevice_stats(struct f2fs_sb_info *sbi) { struct f2fs_stat_info *si = F2FS_STAT(sbi); struct f2fs_dev_stats *dev_stats = si->dev_stats; int i, j; if (!f2fs_is_multi_device(sbi)) return; memset(dev_stats, 0, sizeof(struct f2fs_dev_stats) * sbi->s_ndevs); for (i = 0; i < sbi->s_ndevs; i++) { unsigned int start_segno, end_segno; block_t start_blk, end_blk; if (i == 0) { start_blk = MAIN_BLKADDR(sbi); end_blk = FDEV(i).end_blk + 1 - SEG0_BLKADDR(sbi); } else { start_blk = FDEV(i).start_blk; end_blk = FDEV(i).end_blk + 1; } start_segno = GET_SEGNO(sbi, start_blk); end_segno = GET_SEGNO(sbi, end_blk); for (j = start_segno; j < end_segno; j++) { unsigned int seg_blks, sec_blks; seg_blks = get_seg_entry(sbi, j)->valid_blocks; /* update segment stats */ if (is_curseg(sbi, j)) dev_stats[i].devstats[0][DEVSTAT_INUSE]++; else if (seg_blks == BLKS_PER_SEG(sbi)) dev_stats[i].devstats[0][DEVSTAT_FULL]++; else if (seg_blks != 0) dev_stats[i].devstats[0][DEVSTAT_DIRTY]++; else if (!test_bit(j, FREE_I(sbi)->free_segmap)) dev_stats[i].devstats[0][DEVSTAT_FREE]++; else dev_stats[i].devstats[0][DEVSTAT_PREFREE]++; if (!__is_large_section(sbi) || (j % SEGS_PER_SEC(sbi)) != 0) continue; sec_blks = get_sec_entry(sbi, j)->valid_blocks; /* update section stats */ if (is_cursec(sbi, GET_SEC_FROM_SEG(sbi, j))) dev_stats[i].devstats[1][DEVSTAT_INUSE]++; else if (sec_blks == BLKS_PER_SEC(sbi)) dev_stats[i].devstats[1][DEVSTAT_FULL]++; else if (sec_blks != 0) dev_stats[i].devstats[1][DEVSTAT_DIRTY]++; else if (!test_bit(GET_SEC_FROM_SEG(sbi, j), FREE_I(sbi)->free_secmap)) dev_stats[i].devstats[1][DEVSTAT_FREE]++; else dev_stats[i].devstats[1][DEVSTAT_PREFREE]++; } } } static void update_general_status(struct f2fs_sb_info *sbi) { struct f2fs_stat_info *si = F2FS_STAT(sbi); struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); int i; /* these will be changed if online resize is done */ si->main_area_segs = le32_to_cpu(raw_super->segment_count_main); si->main_area_sections = le32_to_cpu(raw_super->section_count); si->main_area_zones = si->main_area_sections / le32_to_cpu(raw_super->secs_per_zone); /* general extent cache stats */ for (i = 0; i < NR_EXTENT_CACHES; i++) { struct extent_tree_info *eti = &sbi->extent_tree[i]; si->hit_cached[i] = atomic64_read(&sbi->read_hit_cached[i]); si->hit_rbtree[i] = atomic64_read(&sbi->read_hit_rbtree[i]); si->total_ext[i] = atomic64_read(&sbi->total_hit_ext[i]); si->hit_total[i] = si->hit_cached[i] + si->hit_rbtree[i]; si->ext_tree[i] = atomic_read(&eti->total_ext_tree); si->zombie_tree[i] = atomic_read(&eti->total_zombie_tree); si->ext_node[i] = atomic_read(&eti->total_ext_node); } /* read extent_cache only */ si->hit_largest = atomic64_read(&sbi->read_hit_largest); si->hit_total[EX_READ] += si->hit_largest; /* block age extent_cache only */ si->allocated_data_blocks = atomic64_read(&sbi->allocated_data_blocks); /* validation check of the segment numbers */ si->ndirty_node = get_pages(sbi, F2FS_DIRTY_NODES); si->ndirty_dent = get_pages(sbi, F2FS_DIRTY_DENTS); si->ndirty_meta = get_pages(sbi, F2FS_DIRTY_META); si->ndirty_data = get_pages(sbi, F2FS_DIRTY_DATA); si->ndirty_qdata = get_pages(sbi, F2FS_DIRTY_QDATA); si->ndirty_imeta = get_pages(sbi, F2FS_DIRTY_IMETA); si->ndirty_dirs = sbi->ndirty_inode[DIR_INODE]; si->ndirty_files = sbi->ndirty_inode[FILE_INODE]; si->ndonate_files = sbi->donate_files; si->nquota_files = sbi->nquota_files; si->ndirty_all = sbi->ndirty_inode[DIRTY_META]; si->aw_cnt = atomic_read(&sbi->atomic_files); si->max_aw_cnt = atomic_read(&sbi->max_aw_cnt); si->nr_dio_read = get_pages(sbi, F2FS_DIO_READ); si->nr_dio_write = get_pages(sbi, F2FS_DIO_WRITE); si->nr_wb_cp_data = get_pages(sbi, F2FS_WB_CP_DATA); si->nr_wb_data = get_pages(sbi, F2FS_WB_DATA); si->nr_rd_data = get_pages(sbi, F2FS_RD_DATA); si->nr_rd_node = get_pages(sbi, F2FS_RD_NODE); si->nr_rd_meta = get_pages(sbi, F2FS_RD_META); if (SM_I(sbi)->fcc_info) { si->nr_flushed = atomic_read(&SM_I(sbi)->fcc_info->issued_flush); si->nr_flushing = atomic_read(&SM_I(sbi)->fcc_info->queued_flush); si->flush_list_empty = llist_empty(&SM_I(sbi)->fcc_info->issue_list); } if (SM_I(sbi)->dcc_info) { si->nr_discarded = atomic_read(&SM_I(sbi)->dcc_info->issued_discard); si->nr_discarding = atomic_read(&SM_I(sbi)->dcc_info->queued_discard); si->nr_discard_cmd = atomic_read(&SM_I(sbi)->dcc_info->discard_cmd_cnt); si->undiscard_blks = SM_I(sbi)->dcc_info->undiscard_blks; } si->nr_issued_ckpt = atomic_read(&sbi->cprc_info.issued_ckpt); si->nr_total_ckpt = atomic_read(&sbi->cprc_info.total_ckpt); si->nr_queued_ckpt = atomic_read(&sbi->cprc_info.queued_ckpt); spin_lock(&sbi->cprc_info.stat_lock); si->cur_ckpt_time = sbi->cprc_info.cur_time; si->peak_ckpt_time = sbi->cprc_info.peak_time; spin_unlock(&sbi->cprc_info.stat_lock); si->total_count = BLKS_TO_SEGS(sbi, (int)sbi->user_block_count); si->rsvd_segs = reserved_segments(sbi); si->overp_segs = overprovision_segments(sbi); si->valid_count = valid_user_blocks(sbi); si->discard_blks = discard_blocks(sbi); si->valid_node_count = valid_node_count(sbi); si->valid_inode_count = valid_inode_count(sbi); si->inline_xattr = atomic_read(&sbi->inline_xattr); si->inline_inode = atomic_read(&sbi->inline_inode); si->inline_dir = atomic_read(&sbi->inline_dir); si->compr_inode = atomic_read(&sbi->compr_inode); si->swapfile_inode = atomic_read(&sbi->swapfile_inode); si->compr_blocks = atomic64_read(&sbi->compr_blocks); si->append = sbi->im[APPEND_INO].ino_num; si->update = sbi->im[UPDATE_INO].ino_num; si->orphans = sbi->im[ORPHAN_INO].ino_num; si->utilization = utilization(sbi); si->free_segs = free_segments(sbi); si->free_secs = free_sections(sbi); si->prefree_count = prefree_segments(sbi); si->dirty_count = dirty_segments(sbi); if (sbi->node_inode) si->node_pages = NODE_MAPPING(sbi)->nrpages; if (sbi->meta_inode) si->meta_pages = META_MAPPING(sbi)->nrpages; #ifdef CONFIG_F2FS_FS_COMPRESSION if (sbi->compress_inode) { si->compress_pages = COMPRESS_MAPPING(sbi)->nrpages; si->compress_page_hit = atomic_read(&sbi->compress_page_hit); } #endif si->nats = NM_I(sbi)->nat_cnt[TOTAL_NAT]; si->dirty_nats = NM_I(sbi)->nat_cnt[DIRTY_NAT]; si->sits = MAIN_SEGS(sbi); si->dirty_sits = SIT_I(sbi)->dirty_sentries; si->free_nids = NM_I(sbi)->nid_cnt[FREE_NID]; si->avail_nids = NM_I(sbi)->available_nids; si->alloc_nids = NM_I(sbi)->nid_cnt[PREALLOC_NID]; si->io_skip_bggc = sbi->io_skip_bggc; si->other_skip_bggc = sbi->other_skip_bggc; si->util_free = (int)(BLKS_TO_SEGS(sbi, free_user_blocks(sbi))) * 100 / (int)(sbi->user_block_count >> sbi->log_blocks_per_seg) / 2; si->util_valid = (int)(BLKS_TO_SEGS(sbi, written_block_count(sbi))) * 100 / (int)(sbi->user_block_count >> sbi->log_blocks_per_seg) / 2; si->util_invalid = 50 - si->util_free - si->util_valid; for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) { struct curseg_info *curseg = CURSEG_I(sbi, i); si->blkoff[i] = curseg->next_blkoff; si->curseg[i] = curseg->segno; si->cursec[i] = GET_SEC_FROM_SEG(sbi, curseg->segno); si->curzone[i] = GET_ZONE_FROM_SEC(sbi, si->cursec[i]); } for (i = META_CP; i < META_MAX; i++) si->meta_count[i] = atomic_read(&sbi->meta_count[i]); for (i = 0; i < NO_CHECK_TYPE; i++) { si->dirty_seg[i] = 0; si->full_seg[i] = 0; si->valid_blks[i] = 0; } for (i = 0; i < MAIN_SEGS(sbi); i++) { int blks = get_seg_entry(sbi, i)->valid_blocks; int type = get_seg_entry(sbi, i)->type; if (!blks) continue; if (blks == BLKS_PER_SEG(sbi)) si->full_seg[type]++; else si->dirty_seg[type]++; si->valid_blks[type] += blks; } update_multidevice_stats(sbi); for (i = 0; i < MAX_CALL_TYPE; i++) si->cp_call_count[i] = atomic_read(&sbi->cp_call_count[i]); for (i = 0; i < 2; i++) { si->segment_count[i] = sbi->segment_count[i]; si->block_count[i] = sbi->block_count[i]; } si->inplace_count = atomic_read(&sbi->inplace_count); } /* * This function calculates memory footprint. */ static void update_mem_info(struct f2fs_sb_info *sbi) { struct f2fs_stat_info *si = F2FS_STAT(sbi); int i; if (si->base_mem) goto get_cache; /* build stat */ si->base_mem = sizeof(struct f2fs_stat_info); /* build superblock */ si->base_mem += sizeof(struct f2fs_sb_info) + sbi->sb->s_blocksize; si->base_mem += 2 * sizeof(struct f2fs_inode_info); si->base_mem += sizeof(*sbi->ckpt); /* build sm */ si->base_mem += sizeof(struct f2fs_sm_info); /* build sit */ si->base_mem += sizeof(struct sit_info); si->base_mem += MAIN_SEGS(sbi) * sizeof(struct seg_entry); si->base_mem += f2fs_bitmap_size(MAIN_SEGS(sbi)); si->base_mem += 2 * SIT_VBLOCK_MAP_SIZE * MAIN_SEGS(sbi); si->base_mem += SIT_VBLOCK_MAP_SIZE * MAIN_SEGS(sbi); si->base_mem += SIT_VBLOCK_MAP_SIZE; if (__is_large_section(sbi)) si->base_mem += MAIN_SECS(sbi) * sizeof(struct sec_entry); si->base_mem += __bitmap_size(sbi, SIT_BITMAP); /* build free segmap */ si->base_mem += sizeof(struct free_segmap_info); si->base_mem += f2fs_bitmap_size(MAIN_SEGS(sbi)); si->base_mem += f2fs_bitmap_size(MAIN_SECS(sbi)); /* build curseg */ si->base_mem += sizeof(struct curseg_info) * NR_CURSEG_TYPE; si->base_mem += PAGE_SIZE * NR_CURSEG_TYPE; /* build dirty segmap */ si->base_mem += sizeof(struct dirty_seglist_info); si->base_mem += NR_DIRTY_TYPE * f2fs_bitmap_size(MAIN_SEGS(sbi)); si->base_mem += f2fs_bitmap_size(MAIN_SECS(sbi)); /* build nm */ si->base_mem += sizeof(struct f2fs_nm_info); si->base_mem += __bitmap_size(sbi, NAT_BITMAP); si->base_mem += F2FS_BLK_TO_BYTES(NM_I(sbi)->nat_bits_blocks); si->base_mem += NM_I(sbi)->nat_blocks * f2fs_bitmap_size(NAT_ENTRY_PER_BLOCK); si->base_mem += NM_I(sbi)->nat_blocks / 8; si->base_mem += NM_I(sbi)->nat_blocks * sizeof(unsigned short); get_cache: si->cache_mem = 0; /* build gc */ if (sbi->gc_thread) si->cache_mem += sizeof(struct f2fs_gc_kthread); /* build merge flush thread */ if (SM_I(sbi)->fcc_info) si->cache_mem += sizeof(struct flush_cmd_control); if (SM_I(sbi)->dcc_info) { si->cache_mem += sizeof(struct discard_cmd_control); si->cache_mem += sizeof(struct discard_cmd) * atomic_read(&SM_I(sbi)->dcc_info->discard_cmd_cnt); } /* free nids */ si->cache_mem += (NM_I(sbi)->nid_cnt[FREE_NID] + NM_I(sbi)->nid_cnt[PREALLOC_NID]) * sizeof(struct free_nid); si->cache_mem += NM_I(sbi)->nat_cnt[TOTAL_NAT] * sizeof(struct nat_entry); si->cache_mem += NM_I(sbi)->nat_cnt[DIRTY_NAT] * sizeof(struct nat_entry_set); for (i = 0; i < MAX_INO_ENTRY; i++) si->cache_mem += sbi->im[i].ino_num * sizeof(struct ino_entry); for (i = 0; i < NR_EXTENT_CACHES; i++) { struct extent_tree_info *eti = &sbi->extent_tree[i]; si->ext_mem[i] = atomic_read(&eti->total_ext_tree) * sizeof(struct extent_tree); si->ext_mem[i] += atomic_read(&eti->total_ext_node) * sizeof(struct extent_node); si->cache_mem += si->ext_mem[i]; } si->page_mem = 0; if (sbi->node_inode) { unsigned long npages = NODE_MAPPING(sbi)->nrpages; si->page_mem += (unsigned long long)npages << PAGE_SHIFT; } if (sbi->meta_inode) { unsigned long npages = META_MAPPING(sbi)->nrpages; si->page_mem += (unsigned long long)npages << PAGE_SHIFT; } #ifdef CONFIG_F2FS_FS_COMPRESSION if (sbi->compress_inode) { unsigned long npages = COMPRESS_MAPPING(sbi)->nrpages; si->page_mem += (unsigned long long)npages << PAGE_SHIFT; } #endif } static const char *s_flag[MAX_SBI_FLAG] = { [SBI_IS_DIRTY] = "fs_dirty", [SBI_IS_CLOSE] = "closing", [SBI_NEED_FSCK] = "need_fsck", [SBI_POR_DOING] = "recovering", [SBI_NEED_SB_WRITE] = "sb_dirty", [SBI_NEED_CP] = "need_cp", [SBI_IS_SHUTDOWN] = "shutdown", [SBI_IS_RECOVERED] = "recovered", [SBI_CP_DISABLED] = "cp_disabled", [SBI_CP_DISABLED_QUICK] = "cp_disabled_quick", [SBI_QUOTA_NEED_FLUSH] = "quota_need_flush", [SBI_QUOTA_SKIP_FLUSH] = "quota_skip_flush", [SBI_QUOTA_NEED_REPAIR] = "quota_need_repair", [SBI_IS_RESIZEFS] = "resizefs", [SBI_IS_FREEZING] = "freezefs", [SBI_IS_WRITABLE] = "writable", [SBI_ENABLE_CHECKPOINT] = "enable_checkpoint", }; static const char *ipu_mode_names[F2FS_IPU_MAX] = { [F2FS_IPU_FORCE] = "FORCE", [F2FS_IPU_SSR] = "SSR", [F2FS_IPU_UTIL] = "UTIL", [F2FS_IPU_SSR_UTIL] = "SSR_UTIL", [F2FS_IPU_FSYNC] = "FSYNC", [F2FS_IPU_ASYNC] = "ASYNC", [F2FS_IPU_NOCACHE] = "NOCACHE", [F2FS_IPU_HONOR_OPU_WRITE] = "HONOR_OPU_WRITE", }; static int stat_show(struct seq_file *s, void *v) { struct f2fs_stat_info *si; int i = 0, j = 0; spin_lock(&f2fs_stat_lock); list_for_each_entry(si, &f2fs_stat_list, stat_list) { struct f2fs_sb_info *sbi = si->sbi; update_general_status(sbi); seq_printf(s, "\n=====[ partition info(%pg). #%d, %s, CP: %s]=====\n", sbi->sb->s_bdev, i++, f2fs_readonly(sbi->sb) ? "RO" : "RW", is_set_ckpt_flags(sbi, CP_DISABLED_FLAG) ? "Disabled" : (f2fs_cp_error(sbi) ? "Error" : "Good")); if (sbi->s_flag) { seq_puts(s, "[SBI:"); for_each_set_bit(j, &sbi->s_flag, MAX_SBI_FLAG) seq_printf(s, " %s", s_flag[j]); seq_puts(s, "]\n"); } seq_printf(s, "[SB: 1] [CP: 2] [SIT: %d] [NAT: %d] ", si->sit_area_segs, si->nat_area_segs); seq_printf(s, "[SSA: %d] [MAIN: %d", si->ssa_area_segs, si->main_area_segs); seq_printf(s, "(OverProv:%d Resv:%d)]\n\n", si->overp_segs, si->rsvd_segs); seq_printf(s, "Current Time Sec: %llu / Mounted Time Sec: %llu\n\n", ktime_get_boottime_seconds(), SIT_I(sbi)->mounted_time); seq_puts(s, "Policy:\n"); seq_puts(s, " - IPU: ["); if (IS_F2FS_IPU_DISABLE(sbi)) { seq_puts(s, " DISABLE"); } else { unsigned long policy = SM_I(sbi)->ipu_policy; for_each_set_bit(j, &policy, F2FS_IPU_MAX) seq_printf(s, " %s", ipu_mode_names[j]); } seq_puts(s, " ]\n\n"); if (test_opt(sbi, DISCARD)) seq_printf(s, "Utilization: %u%% (%u valid blocks, %u discard blocks)\n", si->utilization, si->valid_count, si->discard_blks); else seq_printf(s, "Utilization: %u%% (%u valid blocks)\n", si->utilization, si->valid_count); seq_printf(s, " - Node: %u (Inode: %u, ", si->valid_node_count, si->valid_inode_count); seq_printf(s, "Other: %u)\n - Data: %u\n", si->valid_node_count - si->valid_inode_count, si->valid_count - si->valid_node_count); seq_printf(s, " - Inline_xattr Inode: %u\n", si->inline_xattr); seq_printf(s, " - Inline_data Inode: %u\n", si->inline_inode); seq_printf(s, " - Inline_dentry Inode: %u\n", si->inline_dir); seq_printf(s, " - Compressed Inode: %u, Blocks: %llu\n", si->compr_inode, si->compr_blocks); seq_printf(s, " - Swapfile Inode: %u\n", si->swapfile_inode); seq_printf(s, " - Donate Inode: %u\n", si->ndonate_files); seq_printf(s, " - Orphan/Append/Update Inode: %u, %u, %u\n", si->orphans, si->append, si->update); seq_printf(s, "\nMain area: %d segs, %d secs %d zones\n", si->main_area_segs, si->main_area_sections, si->main_area_zones); seq_printf(s, " TYPE %8s %8s %8s %8s %10s %10s %10s\n", "blkoff", "segno", "secno", "zoneno", "dirty_seg", "full_seg", "valid_blk"); seq_printf(s, " - COLD data: %8d %8d %8d %8d %10u %10u %10u\n", si->blkoff[CURSEG_COLD_DATA], si->curseg[CURSEG_COLD_DATA], si->cursec[CURSEG_COLD_DATA], si->curzone[CURSEG_COLD_DATA], si->dirty_seg[CURSEG_COLD_DATA], si->full_seg[CURSEG_COLD_DATA], si->valid_blks[CURSEG_COLD_DATA]); seq_printf(s, " - WARM data: %8d %8d %8d %8d %10u %10u %10u\n", si->blkoff[CURSEG_WARM_DATA], si->curseg[CURSEG_WARM_DATA], si->cursec[CURSEG_WARM_DATA], si->curzone[CURSEG_WARM_DATA], si->dirty_seg[CURSEG_WARM_DATA], si->full_seg[CURSEG_WARM_DATA], si->valid_blks[CURSEG_WARM_DATA]); seq_printf(s, " - HOT data: %8d %8d %8d %8d %10u %10u %10u\n", si->blkoff[CURSEG_HOT_DATA], si->curseg[CURSEG_HOT_DATA], si->cursec[CURSEG_HOT_DATA], si->curzone[CURSEG_HOT_DATA], si->dirty_seg[CURSEG_HOT_DATA], si->full_seg[CURSEG_HOT_DATA], si->valid_blks[CURSEG_HOT_DATA]); seq_printf(s, " - Dir dnode: %8d %8d %8d %8d %10u %10u %10u\n", si->blkoff[CURSEG_HOT_NODE], si->curseg[CURSEG_HOT_NODE], si->cursec[CURSEG_HOT_NODE], si->curzone[CURSEG_HOT_NODE], si->dirty_seg[CURSEG_HOT_NODE], si->full_seg[CURSEG_HOT_NODE], si->valid_blks[CURSEG_HOT_NODE]); seq_printf(s, " - File dnode: %8d %8d %8d %8d %10u %10u %10u\n", si->blkoff[CURSEG_WARM_NODE], si->curseg[CURSEG_WARM_NODE], si->cursec[CURSEG_WARM_NODE], si->curzone[CURSEG_WARM_NODE], si->dirty_seg[CURSEG_WARM_NODE], si->full_seg[CURSEG_WARM_NODE], si->valid_blks[CURSEG_WARM_NODE]); seq_printf(s, " - Indir nodes: %8d %8d %8d %8d %10u %10u %10u\n", si->blkoff[CURSEG_COLD_NODE], si->curseg[CURSEG_COLD_NODE], si->cursec[CURSEG_COLD_NODE], si->curzone[CURSEG_COLD_NODE], si->dirty_seg[CURSEG_COLD_NODE], si->full_seg[CURSEG_COLD_NODE], si->valid_blks[CURSEG_COLD_NODE]); seq_printf(s, " - Pinned file: %8d %8d %8d %8d\n", si->blkoff[CURSEG_COLD_DATA_PINNED], si->curseg[CURSEG_COLD_DATA_PINNED], si->cursec[CURSEG_COLD_DATA_PINNED], si->curzone[CURSEG_COLD_DATA_PINNED]); seq_printf(s, " - ATGC data: %8d %8d %8d %8d\n", si->blkoff[CURSEG_ALL_DATA_ATGC], si->curseg[CURSEG_ALL_DATA_ATGC], si->cursec[CURSEG_ALL_DATA_ATGC], si->curzone[CURSEG_ALL_DATA_ATGC]); seq_printf(s, "\n - Valid: %d\n - Dirty: %d\n", si->main_area_segs - si->dirty_count - si->prefree_count - si->free_segs, si->dirty_count); seq_printf(s, " - Prefree: %d\n - Free: %d (%d)\n\n", si->prefree_count, si->free_segs, si->free_secs); if (f2fs_is_multi_device(sbi)) { seq_puts(s, "Multidevice stats:\n"); seq_printf(s, " [seg: %8s %8s %8s %8s %8s]", "inuse", "dirty", "full", "free", "prefree"); if (__is_large_section(sbi)) seq_printf(s, " [sec: %8s %8s %8s %8s %8s]\n", "inuse", "dirty", "full", "free", "prefree"); else seq_puts(s, "\n"); for (i = 0; i < sbi->s_ndevs; i++) { seq_printf(s, " #%-2d %8u %8u %8u %8u %8u", i, si->dev_stats[i].devstats[0][DEVSTAT_INUSE], si->dev_stats[i].devstats[0][DEVSTAT_DIRTY], si->dev_stats[i].devstats[0][DEVSTAT_FULL], si->dev_stats[i].devstats[0][DEVSTAT_FREE], si->dev_stats[i].devstats[0][DEVSTAT_PREFREE]); if (!__is_large_section(sbi)) { seq_puts(s, "\n"); continue; } seq_printf(s, " %8u %8u %8u %8u %8u\n", si->dev_stats[i].devstats[1][DEVSTAT_INUSE], si->dev_stats[i].devstats[1][DEVSTAT_DIRTY], si->dev_stats[i].devstats[1][DEVSTAT_FULL], si->dev_stats[i].devstats[1][DEVSTAT_FREE], si->dev_stats[i].devstats[1][DEVSTAT_PREFREE]); } seq_puts(s, "\n"); } seq_printf(s, "CP calls: %d (BG: %d)\n", si->cp_call_count[TOTAL_CALL], si->cp_call_count[BACKGROUND]); seq_printf(s, "CP count: %d\n", si->cp_count); seq_printf(s, " - cp blocks : %u\n", si->meta_count[META_CP]); seq_printf(s, " - sit blocks : %u\n", si->meta_count[META_SIT]); seq_printf(s, " - nat blocks : %u\n", si->meta_count[META_NAT]); seq_printf(s, " - ssa blocks : %u\n", si->meta_count[META_SSA]); seq_puts(s, "CP merge:\n"); seq_printf(s, " - Queued : %4d\n", si->nr_queued_ckpt); seq_printf(s, " - Issued : %4d\n", si->nr_issued_ckpt); seq_printf(s, " - Total : %4d\n", si->nr_total_ckpt); seq_printf(s, " - Cur time : %4d(ms)\n", si->cur_ckpt_time); seq_printf(s, " - Peak time : %4d(ms)\n", si->peak_ckpt_time); seq_printf(s, "GC calls: %d (gc_thread: %d)\n", si->gc_call_count[BACKGROUND] + si->gc_call_count[FOREGROUND], si->gc_call_count[BACKGROUND]); if (__is_large_section(sbi)) { seq_printf(s, " - data sections : %d (BG: %d)\n", si->gc_secs[DATA][BG_GC] + si->gc_secs[DATA][FG_GC], si->gc_secs[DATA][BG_GC]); seq_printf(s, " - node sections : %d (BG: %d)\n", si->gc_secs[NODE][BG_GC] + si->gc_secs[NODE][FG_GC], si->gc_secs[NODE][BG_GC]); } seq_printf(s, " - data segments : %d (BG: %d)\n", si->gc_segs[DATA][BG_GC] + si->gc_segs[DATA][FG_GC], si->gc_segs[DATA][BG_GC]); seq_printf(s, " - node segments : %d (BG: %d)\n", si->gc_segs[NODE][BG_GC] + si->gc_segs[NODE][FG_GC], si->gc_segs[NODE][BG_GC]); seq_puts(s, " - Reclaimed segs :\n"); seq_printf(s, " - Normal : %d\n", sbi->gc_reclaimed_segs[GC_NORMAL]); seq_printf(s, " - Idle CB : %d\n", sbi->gc_reclaimed_segs[GC_IDLE_CB]); seq_printf(s, " - Idle Greedy : %d\n", sbi->gc_reclaimed_segs[GC_IDLE_GREEDY]); seq_printf(s, " - Idle AT : %d\n", sbi->gc_reclaimed_segs[GC_IDLE_AT]); seq_printf(s, " - Urgent High : %d\n", sbi->gc_reclaimed_segs[GC_URGENT_HIGH]); seq_printf(s, " - Urgent Mid : %d\n", sbi->gc_reclaimed_segs[GC_URGENT_MID]); seq_printf(s, " - Urgent Low : %d\n", sbi->gc_reclaimed_segs[GC_URGENT_LOW]); seq_printf(s, "Try to move %d blocks (BG: %d)\n", si->tot_blks, si->bg_data_blks + si->bg_node_blks); seq_printf(s, " - data blocks : %d (%d)\n", si->data_blks, si->bg_data_blks); seq_printf(s, " - node blocks : %d (%d)\n", si->node_blks, si->bg_node_blks); seq_printf(s, "BG skip : IO: %u, Other: %u\n", si->io_skip_bggc, si->other_skip_bggc); seq_printf(s, "defrag blocks : %u\n", si->defrag_blks); seq_puts(s, "\nExtent Cache (Read):\n"); seq_printf(s, " - Hit Count: L1-1:%llu L1-2:%llu L2:%llu\n", si->hit_largest, si->hit_cached[EX_READ], si->hit_rbtree[EX_READ]); seq_printf(s, " - Hit Ratio: %llu%% (%llu / %llu)\n", !si->total_ext[EX_READ] ? 0 : div64_u64(si->hit_total[EX_READ] * 100, si->total_ext[EX_READ]), si->hit_total[EX_READ], si->total_ext[EX_READ]); seq_printf(s, " - Inner Struct Count: tree: %d(%d), node: %d\n", si->ext_tree[EX_READ], si->zombie_tree[EX_READ], si->ext_node[EX_READ]); seq_puts(s, "\nExtent Cache (Block Age):\n"); seq_printf(s, " - Allocated Data Blocks: %llu\n", si->allocated_data_blocks); seq_printf(s, " - Hit Count: L1:%llu L2:%llu\n", si->hit_cached[EX_BLOCK_AGE], si->hit_rbtree[EX_BLOCK_AGE]); seq_printf(s, " - Hit Ratio: %llu%% (%llu / %llu)\n", !si->total_ext[EX_BLOCK_AGE] ? 0 : div64_u64(si->hit_total[EX_BLOCK_AGE] * 100, si->total_ext[EX_BLOCK_AGE]), si->hit_total[EX_BLOCK_AGE], si->total_ext[EX_BLOCK_AGE]); seq_printf(s, " - Inner Struct Count: tree: %d(%d), node: %d\n", si->ext_tree[EX_BLOCK_AGE], si->zombie_tree[EX_BLOCK_AGE], si->ext_node[EX_BLOCK_AGE]); seq_puts(s, "\nBalancing F2FS Async:\n"); seq_printf(s, " - DIO (R: %4d, W: %4d)\n", si->nr_dio_read, si->nr_dio_write); seq_printf(s, " - IO_R (Data: %4d, Node: %4d, Meta: %4d\n", si->nr_rd_data, si->nr_rd_node, si->nr_rd_meta); seq_printf(s, " - IO_W (CP: %4d, Data: %4d, Flush: (%4d %4d %4d), ", si->nr_wb_cp_data, si->nr_wb_data, si->nr_flushing, si->nr_flushed, si->flush_list_empty); seq_printf(s, "Discard: (%4d %4d)) cmd: %4d undiscard:%4u\n", si->nr_discarding, si->nr_discarded, si->nr_discard_cmd, si->undiscard_blks); seq_printf(s, " - atomic IO: %4d (Max. %4d)\n", si->aw_cnt, si->max_aw_cnt); seq_printf(s, " - compress: %4d, hit:%8d\n", si->compress_pages, si->compress_page_hit); seq_printf(s, " - nodes: %4d in %4d\n", si->ndirty_node, si->node_pages); seq_printf(s, " - dents: %4d in dirs:%4d (%4d)\n", si->ndirty_dent, si->ndirty_dirs, si->ndirty_all); seq_printf(s, " - data: %4d in files:%4d\n", si->ndirty_data, si->ndirty_files); seq_printf(s, " - quota data: %4d in quota files:%4d\n", si->ndirty_qdata, si->nquota_files); seq_printf(s, " - meta: %4d in %4d\n", si->ndirty_meta, si->meta_pages); seq_printf(s, " - imeta: %4d\n", si->ndirty_imeta); seq_printf(s, " - fsync mark: %4lld\n", percpu_counter_sum_positive( &sbi->rf_node_block_count)); seq_printf(s, " - NATs: %9d/%9d\n - SITs: %9d/%9d\n", si->dirty_nats, si->nats, si->dirty_sits, si->sits); seq_printf(s, " - free_nids: %9d/%9d\n - alloc_nids: %9d\n", si->free_nids, si->avail_nids, si->alloc_nids); seq_puts(s, "\nDistribution of User Blocks:"); seq_puts(s, " [ valid | invalid | free ]\n"); seq_puts(s, " ["); for (j = 0; j < si->util_valid; j++) seq_putc(s, '-'); seq_putc(s, '|'); for (j = 0; j < si->util_invalid; j++) seq_putc(s, '-'); seq_putc(s, '|'); for (j = 0; j < si->util_free; j++) seq_putc(s, '-'); seq_puts(s, "]\n\n"); seq_printf(s, "IPU: %u blocks\n", si->inplace_count); seq_printf(s, "SSR: %u blocks in %u segments\n", si->block_count[SSR], si->segment_count[SSR]); seq_printf(s, "LFS: %u blocks in %u segments\n", si->block_count[LFS], si->segment_count[LFS]); /* segment usage info */ f2fs_update_sit_info(sbi); seq_printf(s, "\nBDF: %u, avg. vblocks: %u\n", si->bimodal, si->avg_vblocks); /* memory footprint */ update_mem_info(sbi); seq_printf(s, "\nMemory: %llu KB\n", (si->base_mem + si->cache_mem + si->page_mem) >> 10); seq_printf(s, " - static: %llu KB\n", si->base_mem >> 10); seq_printf(s, " - cached all: %llu KB\n", si->cache_mem >> 10); seq_printf(s, " - read extent cache: %llu KB\n", si->ext_mem[EX_READ] >> 10); seq_printf(s, " - block age extent cache: %llu KB\n", si->ext_mem[EX_BLOCK_AGE] >> 10); seq_printf(s, " - paged : %llu KB\n", si->page_mem >> 10); } spin_unlock(&f2fs_stat_lock); return 0; } DEFINE_SHOW_ATTRIBUTE(stat); #endif int f2fs_build_stats(struct f2fs_sb_info *sbi) { struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi); struct f2fs_stat_info *si; struct f2fs_dev_stats *dev_stats; int i; si = f2fs_kzalloc(sbi, sizeof(struct f2fs_stat_info), GFP_KERNEL); if (!si) return -ENOMEM; dev_stats = f2fs_kzalloc(sbi, sizeof(struct f2fs_dev_stats) * sbi->s_ndevs, GFP_KERNEL); if (!dev_stats) { kfree(si); return -ENOMEM; } si->dev_stats = dev_stats; si->all_area_segs = le32_to_cpu(raw_super->segment_count); si->sit_area_segs = le32_to_cpu(raw_super->segment_count_sit); si->nat_area_segs = le32_to_cpu(raw_super->segment_count_nat); si->ssa_area_segs = le32_to_cpu(raw_super->segment_count_ssa); si->main_area_segs = le32_to_cpu(raw_super->segment_count_main); si->main_area_sections = le32_to_cpu(raw_super->section_count); si->main_area_zones = si->main_area_sections / le32_to_cpu(raw_super->secs_per_zone); si->sbi = sbi; sbi->stat_info = si; /* general extent cache stats */ for (i = 0; i < NR_EXTENT_CACHES; i++) { atomic64_set(&sbi->total_hit_ext[i], 0); atomic64_set(&sbi->read_hit_rbtree[i], 0); atomic64_set(&sbi->read_hit_cached[i], 0); } /* read extent_cache only */ atomic64_set(&sbi->read_hit_largest, 0); atomic_set(&sbi->inline_xattr, 0); atomic_set(&sbi->inline_inode, 0); atomic_set(&sbi->inline_dir, 0); atomic_set(&sbi->compr_inode, 0); atomic64_set(&sbi->compr_blocks, 0); atomic_set(&sbi->swapfile_inode, 0); atomic_set(&sbi->atomic_files, 0); atomic_set(&sbi->inplace_count, 0); for (i = META_CP; i < META_MAX; i++) atomic_set(&sbi->meta_count[i], 0); for (i = 0; i < MAX_CALL_TYPE; i++) atomic_set(&sbi->cp_call_count[i], 0); atomic_set(&sbi->max_aw_cnt, 0); spin_lock(&f2fs_stat_lock); list_add_tail(&si->stat_list, &f2fs_stat_list); spin_unlock(&f2fs_stat_lock); return 0; } void f2fs_destroy_stats(struct f2fs_sb_info *sbi) { struct f2fs_stat_info *si = F2FS_STAT(sbi); spin_lock(&f2fs_stat_lock); list_del(&si->stat_list); spin_unlock(&f2fs_stat_lock); kfree(si->dev_stats); kfree(si); } void __init f2fs_create_root_stats(void) { #ifdef CONFIG_DEBUG_FS f2fs_debugfs_root = debugfs_create_dir("f2fs", NULL); debugfs_create_file("status", 0444, f2fs_debugfs_root, NULL, &stat_fops); #endif } void f2fs_destroy_root_stats(void) { #ifdef CONFIG_DEBUG_FS debugfs_remove_recursive(f2fs_debugfs_root); f2fs_debugfs_root = NULL; #endif } |
| 4118 1082 3345 3343 353 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 | /* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright 2019 Google LLC */ #ifndef __LINUX_BLK_CRYPTO_H #define __LINUX_BLK_CRYPTO_H #include <linux/minmax.h> #include <linux/types.h> #include <uapi/linux/blk-crypto.h> enum blk_crypto_mode_num { BLK_ENCRYPTION_MODE_INVALID, BLK_ENCRYPTION_MODE_AES_256_XTS, BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV, BLK_ENCRYPTION_MODE_ADIANTUM, BLK_ENCRYPTION_MODE_SM4_XTS, BLK_ENCRYPTION_MODE_MAX, }; /* * Supported types of keys. Must be bitflags due to their use in * blk_crypto_profile::key_types_supported. */ enum blk_crypto_key_type { /* * Raw keys (i.e. "software keys"). These keys are simply kept in raw, * plaintext form in kernel memory. */ BLK_CRYPTO_KEY_TYPE_RAW = 0x1, /* * Hardware-wrapped keys. These keys are only present in kernel memory * in ephemerally-wrapped form, and they can only be unwrapped by * dedicated hardware. For details, see the "Hardware-wrapped keys" * section of Documentation/block/inline-encryption.rst. */ BLK_CRYPTO_KEY_TYPE_HW_WRAPPED = 0x2, }; /* * Currently the maximum raw key size is 64 bytes, as that is the key size of * BLK_ENCRYPTION_MODE_AES_256_XTS which takes the longest key. * * The maximum hardware-wrapped key size depends on the hardware's key wrapping * algorithm, which is a hardware implementation detail, so it isn't precisely * specified. But currently 128 bytes is plenty in practice. Implementations * are recommended to wrap a 32-byte key for the hardware KDF with AES-256-GCM, * which should result in a size closer to 64 bytes than 128. * * Both of these values can trivially be increased if ever needed. */ #define BLK_CRYPTO_MAX_RAW_KEY_SIZE 64 #define BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE 128 #define BLK_CRYPTO_MAX_ANY_KEY_SIZE \ MAX(BLK_CRYPTO_MAX_RAW_KEY_SIZE, BLK_CRYPTO_MAX_HW_WRAPPED_KEY_SIZE) /* * Size of the "software secret" which can be derived from a hardware-wrapped * key. This is currently always 32 bytes. Note, the choice of 32 bytes * assumes that the software secret is only used directly for algorithms that * don't require more than a 256-bit key to get the desired security strength. * If it were to be used e.g. directly as an AES-256-XTS key, then this would * need to be increased (which is possible if hardware supports it, but care * would need to be taken to avoid breaking users who need exactly 32 bytes). */ #define BLK_CRYPTO_SW_SECRET_SIZE 32 /** * struct blk_crypto_config - an inline encryption key's crypto configuration * @crypto_mode: encryption algorithm this key is for * @data_unit_size: the data unit size for all encryption/decryptions with this * key. This is the size in bytes of each individual plaintext and * ciphertext. This is always a power of 2. It might be e.g. the * filesystem block size or the disk sector size. * @dun_bytes: the maximum number of bytes of DUN used when using this key * @key_type: the type of this key -- either raw or hardware-wrapped */ struct blk_crypto_config { enum blk_crypto_mode_num crypto_mode; unsigned int data_unit_size; unsigned int dun_bytes; enum blk_crypto_key_type key_type; }; /** * struct blk_crypto_key - an inline encryption key * @crypto_cfg: the crypto mode, data unit size, key type, and other * characteristics of this key and how it will be used * @data_unit_size_bits: log2 of data_unit_size * @size: size of this key in bytes. The size of a raw key is fixed for a given * crypto mode, but the size of a hardware-wrapped key can vary. * @bytes: the bytes of this key. Only the first @size bytes are significant. * * A blk_crypto_key is immutable once created, and many bios can reference it at * the same time. It must not be freed until all bios using it have completed * and it has been evicted from all devices on which it may have been used. */ struct blk_crypto_key { struct blk_crypto_config crypto_cfg; unsigned int data_unit_size_bits; unsigned int size; u8 bytes[BLK_CRYPTO_MAX_ANY_KEY_SIZE]; }; #define BLK_CRYPTO_MAX_IV_SIZE 32 #define BLK_CRYPTO_DUN_ARRAY_SIZE (BLK_CRYPTO_MAX_IV_SIZE / sizeof(u64)) /** * struct bio_crypt_ctx - an inline encryption context * @bc_key: the key, algorithm, and data unit size to use * @bc_dun: the data unit number (starting IV) to use * * A bio_crypt_ctx specifies that the contents of the bio will be encrypted (for * write requests) or decrypted (for read requests) inline by the storage device * or controller, or by the crypto API fallback. */ struct bio_crypt_ctx { const struct blk_crypto_key *bc_key; u64 bc_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; }; #include <linux/blk_types.h> #include <linux/blkdev.h> #ifdef CONFIG_BLK_INLINE_ENCRYPTION static inline bool bio_has_crypt_ctx(struct bio *bio) { return bio->bi_crypt_context; } static inline struct bio_crypt_ctx *bio_crypt_ctx(struct bio *bio) { return bio->bi_crypt_context; } void bio_crypt_set_ctx(struct bio *bio, const struct blk_crypto_key *key, const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], gfp_t gfp_mask); bool bio_crypt_dun_is_contiguous(const struct bio_crypt_ctx *bc, unsigned int bytes, const u64 next_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]); int blk_crypto_init_key(struct blk_crypto_key *blk_key, const u8 *key_bytes, size_t key_size, enum blk_crypto_key_type key_type, enum blk_crypto_mode_num crypto_mode, unsigned int dun_bytes, unsigned int data_unit_size); int blk_crypto_start_using_key(struct block_device *bdev, const struct blk_crypto_key *key); void blk_crypto_evict_key(struct block_device *bdev, const struct blk_crypto_key *key); bool blk_crypto_config_supported_natively(struct block_device *bdev, const struct blk_crypto_config *cfg); bool blk_crypto_config_supported(struct block_device *bdev, const struct blk_crypto_config *cfg); int blk_crypto_derive_sw_secret(struct block_device *bdev, const u8 *eph_key, size_t eph_key_size, u8 sw_secret[BLK_CRYPTO_SW_SECRET_SIZE]); #else /* CONFIG_BLK_INLINE_ENCRYPTION */ static inline bool bio_has_crypt_ctx(struct bio *bio) { return false; } static inline struct bio_crypt_ctx *bio_crypt_ctx(struct bio *bio) { return NULL; } #endif /* CONFIG_BLK_INLINE_ENCRYPTION */ bool __blk_crypto_submit_bio(struct bio *bio); /** * blk_crypto_submit_bio - Submit a bio that may have a crypto context * @bio: bio to submit * * If @bio has no crypto context, or the crypt context attached to @bio is * supported by the underlying device's inline encryption hardware, just submit * @bio. * * Otherwise, try to perform en/decryption for this bio by falling back to the * kernel crypto API. For encryption this means submitting newly allocated * bios for the encrypted payload while keeping back the source bio until they * complete, while for reads the decryption happens in-place by a hooked in * completion handler. */ static inline void blk_crypto_submit_bio(struct bio *bio) { if (!bio_has_crypt_ctx(bio) || __blk_crypto_submit_bio(bio)) submit_bio(bio); } int __bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask); /** * bio_crypt_clone - clone bio encryption context * @dst: destination bio * @src: source bio * @gfp_mask: memory allocation flags * * If @src has an encryption context, clone it to @dst. * * Return: 0 on success, -ENOMEM if out of memory. -ENOMEM is only possible if * @gfp_mask doesn't include %__GFP_DIRECT_RECLAIM. */ static inline int bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask) { if (bio_has_crypt_ctx(src)) return __bio_crypt_clone(dst, src, gfp_mask); return 0; } #endif /* __LINUX_BLK_CRYPTO_H */ |
| 3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 | /* SPDX-License-Identifier: GPL-2.0 */ /* * Shared Memory Communications over RDMA (SMC-R) and RoCE * * Definitions for SMC Connections, Link Groups and Links * * Copyright IBM Corp. 2016 * * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> */ #ifndef _SMC_CORE_H #define _SMC_CORE_H #include <linux/atomic.h> #include <linux/types.h> #include <linux/smc.h> #include <linux/pci.h> #include <rdma/ib_verbs.h> #include <net/genetlink.h> #include <net/smc.h> #include "smc.h" #include "smc_ib.h" #include "smc_clc.h" #define SMC_RMBS_PER_LGR_MAX 255 /* max. # of RMBs per link group */ #define SMC_CONN_PER_LGR_MIN 16 /* min. # of connections per link group */ #define SMC_CONN_PER_LGR_MAX 255 /* max. # of connections per link group, * also is the default value for SMC-R v1 and v2.0 */ #define SMC_CONN_PER_LGR_PREFER 255 /* Preferred connections per link group used for * SMC-R v2.1 and later negotiation, vendors or * distributions may modify it to a value between * 16-255 as needed. */ #define SMCR_MAX_SEND_WR_DEF 16 /* Default number of work requests per send queue */ #define SMCR_MAX_RECV_WR_DEF 48 /* Default number of work requests per recv queue */ struct smc_lgr_list { /* list of link group definition */ struct list_head list; spinlock_t lock; /* protects list of link groups */ u32 num; /* unique link group number */ }; enum smc_lgr_role { /* possible roles of a link group */ SMC_CLNT, /* client */ SMC_SERV /* server */ }; enum smc_link_state { /* possible states of a link */ SMC_LNK_UNUSED, /* link is unused */ SMC_LNK_INACTIVE, /* link is inactive */ SMC_LNK_ACTIVATING, /* link is being activated */ SMC_LNK_ACTIVE, /* link is active */ }; #define SMC_WR_BUF_SIZE 48 /* size of work request buffer */ #define SMC_WR_BUF_V2_SIZE 8192 /* size of v2 work request buffer */ struct smc_wr_buf { u8 raw[SMC_WR_BUF_SIZE]; }; struct smc_wr_v2_buf { u8 raw[SMC_WR_BUF_V2_SIZE]; }; #define SMC_WR_REG_MR_WAIT_TIME (5 * HZ)/* wait time for ib_wr_reg_mr result */ enum smc_wr_reg_state { POSTED, /* ib_wr_reg_mr request posted */ CONFIRMED, /* ib_wr_reg_mr response: successful */ FAILED /* ib_wr_reg_mr response: failure */ }; struct smc_rdma_sge { /* sges for RDMA writes */ struct ib_sge wr_tx_rdma_sge[SMC_IB_MAX_SEND_SGE]; }; #define SMC_MAX_RDMA_WRITES 2 /* max. # of RDMA writes per * message send */ struct smc_rdma_sges { /* sges per message send */ struct smc_rdma_sge tx_rdma_sge[SMC_MAX_RDMA_WRITES]; }; struct smc_rdma_wr { /* work requests per message * send */ struct ib_rdma_wr wr_tx_rdma[SMC_MAX_RDMA_WRITES]; }; #define SMC_LGR_ID_SIZE 4 struct smc_link { struct smc_ib_device *smcibdev; /* ib-device */ u8 ibport; /* port - values 1 | 2 */ struct ib_pd *roce_pd; /* IB protection domain, * unique for every RoCE QP */ struct ib_qp *roce_qp; /* IB queue pair */ struct ib_qp_attr qp_attr; /* IB queue pair attributes */ struct smc_wr_buf *wr_tx_bufs; /* WR send payload buffers */ struct ib_send_wr *wr_tx_ibs; /* WR send meta data */ struct ib_sge *wr_tx_sges; /* WR send gather meta data */ struct smc_rdma_sges *wr_tx_rdma_sges;/*RDMA WRITE gather meta data*/ struct smc_rdma_wr *wr_tx_rdmas; /* WR RDMA WRITE */ struct smc_wr_tx_pend *wr_tx_pends; /* WR send waiting for CQE */ struct completion *wr_tx_compl; /* WR send CQE completion */ /* above four vectors have wr_tx_cnt elements and use the same index */ struct ib_send_wr *wr_tx_v2_ib; /* WR send v2 meta data */ struct ib_sge *wr_tx_v2_sge; /* WR send v2 gather meta data*/ struct smc_wr_tx_pend *wr_tx_v2_pend; /* WR send v2 waiting for CQE */ dma_addr_t wr_tx_dma_addr; /* DMA address of wr_tx_bufs */ dma_addr_t wr_tx_v2_dma_addr; /* DMA address of v2 tx buf*/ atomic_long_t wr_tx_id; /* seq # of last sent WR */ unsigned long *wr_tx_mask; /* bit mask of used indexes */ u32 wr_tx_cnt; /* number of WR send buffers */ wait_queue_head_t wr_tx_wait; /* wait for free WR send buf */ struct { struct percpu_ref wr_tx_refs; } ____cacheline_aligned_in_smp; struct completion tx_ref_comp; u8 *wr_rx_bufs; /* WR recv payload buffers */ struct ib_recv_wr *wr_rx_ibs; /* WR recv meta data */ struct ib_sge *wr_rx_sges; /* WR recv scatter meta data */ /* above three vectors have wr_rx_cnt elements and use the same index */ int wr_rx_sge_cnt; /* rx sge, V1 is 1, V2 is either 2 or 1 */ int wr_rx_buflen; /* buffer len for the first sge, len for the * second sge is lgr shared if rx sge is 2. */ dma_addr_t wr_rx_dma_addr; /* DMA address of wr_rx_bufs */ dma_addr_t wr_rx_v2_dma_addr; /* DMA address of v2 rx buf*/ u64 wr_rx_id; /* seq # of last recv WR */ u64 wr_rx_id_compl; /* seq # of last completed WR */ u32 wr_rx_cnt; /* number of WR recv buffers */ unsigned long wr_rx_tstamp; /* jiffies when last buf rx */ wait_queue_head_t wr_rx_empty_wait; /* wait for RQ empty */ struct ib_reg_wr wr_reg; /* WR register memory region */ wait_queue_head_t wr_reg_wait; /* wait for wr_reg result */ struct { struct percpu_ref wr_reg_refs; } ____cacheline_aligned_in_smp; struct completion reg_ref_comp; enum smc_wr_reg_state wr_reg_state; /* state of wr_reg request */ u8 gid[SMC_GID_SIZE];/* gid matching used vlan id*/ u8 sgid_index; /* gid index for vlan id */ u32 peer_qpn; /* QP number of peer */ enum ib_mtu path_mtu; /* used mtu */ enum ib_mtu peer_mtu; /* mtu size of peer */ u32 psn_initial; /* QP tx initial packet seqno */ u32 peer_psn; /* QP rx initial packet seqno */ u8 peer_mac[ETH_ALEN]; /* = gid[8:10||13:15] */ u8 peer_gid[SMC_GID_SIZE]; /* gid of peer*/ u8 link_id; /* unique # within link group */ u8 link_uid[SMC_LGR_ID_SIZE]; /* unique lnk id */ u8 peer_link_uid[SMC_LGR_ID_SIZE]; /* peer uid */ u8 link_idx; /* index in lgr link array */ u8 link_is_asym; /* is link asymmetric? */ u8 clearing : 1; /* link is being cleared */ refcount_t refcnt; /* link reference count */ struct smc_link_group *lgr; /* parent link group */ struct work_struct link_down_wrk; /* wrk to bring link down */ char ibname[IB_DEVICE_NAME_MAX]; /* ib device name */ int ndev_ifidx; /* network device ifindex */ enum smc_link_state state; /* state of link */ struct delayed_work llc_testlink_wrk; /* testlink worker */ struct completion llc_testlink_resp; /* wait for rx of testlink */ int llc_testlink_time; /* testlink interval */ atomic_t conn_cnt; /* connections on this link */ u16 max_send_wr; u16 max_recv_wr; }; /* For now we just allow one parallel link per link group. The SMC protocol * allows more (up to 8). */ #define SMC_LINKS_PER_LGR_MAX 3 #define SMC_SINGLE_LINK 0 #define SMC_LINKS_ADD_LNK_MIN 1 /* min. # of links per link group */ #define SMC_LINKS_ADD_LNK_MAX 2 /* max. # of links per link group, also is the * default value for smc-r v1.0 and v2.0 */ #define SMC_LINKS_PER_LGR_MAX_PREFER 2 /* Preferred max links per link group used for * SMC-R v2.1 and later negotiation, vendors or * distributions may modify it to a value between * 1-2 as needed. */ /* tx/rx buffer list element for sndbufs list and rmbs list of a lgr */ struct smc_buf_desc { struct list_head list; void *cpu_addr; /* virtual address of buffer */ struct page *pages; int len; /* length of buffer */ u32 used; /* currently used / unused */ union { struct { /* SMC-R */ struct sg_table sgt[SMC_LINKS_PER_LGR_MAX]; /* virtual buffer */ struct ib_mr *mr[SMC_LINKS_PER_LGR_MAX]; /* memory region: for rmb and * vzalloced sndbuf * incl. rkey provided to peer * and lkey provided to local */ u32 order; /* allocation order */ u8 is_conf_rkey; /* confirm_rkey done */ u8 is_reg_mr[SMC_LINKS_PER_LGR_MAX]; /* mem region registered */ u8 is_map_ib[SMC_LINKS_PER_LGR_MAX]; /* mem region mapped to lnk */ u8 is_dma_need_sync; u8 is_reg_err; /* buffer registration err */ u8 is_vm; /* virtually contiguous */ }; struct { /* SMC-D */ /* SMC-D tx buffer */ bool is_attached; /* no need for explicit writes */ /* SMC-D rx buffer: */ unsigned short sba_idx; /* SBA index number */ u64 token; /* DMB token number */ dma_addr_t dma_addr; /* DMA address */ }; }; }; struct smc_rtoken { /* address/key of remote RMB */ u64 dma_addr; u32 rkey; }; #define SMC_BUF_MIN_SIZE 16384 /* minimum size of an RMB */ #define SMC_RMBE_SIZES 16 /* number of distinct RMBE sizes */ /* theoretically, the RFC states that largest size would be 512K, * i.e. compressed 5 and thus 6 sizes (0..5), despite * struct smc_clc_msg_accept_confirm.rmbe_size being a 4 bit value (0..15) */ struct smcd_dev; enum smc_lgr_type { /* redundancy state of lgr */ SMC_LGR_NONE, /* no active links, lgr to be deleted */ SMC_LGR_SINGLE, /* 1 active RNIC on each peer */ SMC_LGR_SYMMETRIC, /* 2 active RNICs on each peer */ SMC_LGR_ASYMMETRIC_PEER, /* local has 2, peer 1 active RNICs */ SMC_LGR_ASYMMETRIC_LOCAL, /* local has 1, peer 2 active RNICs */ }; enum smcr_buf_type { /* types of SMC-R sndbufs and RMBs */ SMCR_PHYS_CONT_BUFS = 0, SMCR_VIRT_CONT_BUFS = 1, SMCR_MIXED_BUFS = 2, }; enum smc_llc_flowtype { SMC_LLC_FLOW_NONE = 0, SMC_LLC_FLOW_ADD_LINK = 2, SMC_LLC_FLOW_DEL_LINK = 4, SMC_LLC_FLOW_REQ_ADD_LINK = 5, SMC_LLC_FLOW_RKEY = 6, }; struct smc_llc_qentry; struct smc_llc_flow { enum smc_llc_flowtype type; struct smc_llc_qentry *qentry; }; struct smc_link_group { struct list_head list; struct rb_root conns_all; /* connection tree */ rwlock_t conns_lock; /* protects conns_all */ unsigned int conns_num; /* current # of connections */ unsigned short vlan_id; /* vlan id of link group */ struct list_head sndbufs[SMC_RMBE_SIZES];/* tx buffers */ struct rw_semaphore sndbufs_lock; /* protects tx buffers */ struct list_head rmbs[SMC_RMBE_SIZES]; /* rx buffers */ struct rw_semaphore rmbs_lock; /* protects rx buffers */ u64 alloc_sndbufs; /* stats of tx buffers */ u64 alloc_rmbs; /* stats of rx buffers */ u8 id[SMC_LGR_ID_SIZE]; /* unique lgr id */ struct delayed_work free_work; /* delayed freeing of an lgr */ struct work_struct terminate_work; /* abnormal lgr termination */ struct workqueue_struct *tx_wq; /* wq for conn. tx workers */ u8 sync_err : 1; /* lgr no longer fits to peer */ u8 terminating : 1;/* lgr is terminating */ u8 freeing : 1; /* lgr is being freed */ refcount_t refcnt; /* lgr reference count */ bool is_smcd; /* SMC-R or SMC-D */ u8 smc_version; u8 negotiated_eid[SMC_MAX_EID_LEN]; u8 peer_os; /* peer operating system */ u8 peer_smc_release; u8 peer_hostname[SMC_MAX_HOSTNAME_LEN]; union { struct { /* SMC-R */ enum smc_lgr_role role; /* client or server */ struct smc_link lnk[SMC_LINKS_PER_LGR_MAX]; /* smc link */ struct smc_wr_v2_buf *wr_rx_buf_v2; /* WR v2 recv payload buffer */ struct smc_wr_v2_buf *wr_tx_buf_v2; /* WR v2 send payload buffer */ char peer_systemid[SMC_SYSTEMID_LEN]; /* unique system_id of peer */ struct smc_rtoken rtokens[SMC_RMBS_PER_LGR_MAX] [SMC_LINKS_PER_LGR_MAX]; /* remote addr/key pairs */ DECLARE_BITMAP(rtokens_used_mask, SMC_RMBS_PER_LGR_MAX); /* used rtoken elements */ u8 next_link_id; enum smc_lgr_type type; enum smcr_buf_type buf_type; /* redundancy state */ u8 pnet_id[SMC_MAX_PNETID_LEN + 1]; /* pnet id of this lgr */ struct list_head llc_event_q; /* queue for llc events */ spinlock_t llc_event_q_lock; /* protects llc_event_q */ struct rw_semaphore llc_conf_mutex; /* protects lgr reconfig. */ struct work_struct llc_add_link_work; struct work_struct llc_del_link_work; struct work_struct llc_event_work; /* llc event worker */ wait_queue_head_t llc_flow_waiter; /* w4 next llc event */ wait_queue_head_t llc_msg_waiter; /* w4 next llc msg */ struct smc_llc_flow llc_flow_lcl; /* llc local control field */ struct smc_llc_flow llc_flow_rmt; /* llc remote control field */ struct smc_llc_qentry *delayed_event; /* arrived when flow active */ spinlock_t llc_flow_lock; /* protects llc flow */ int llc_testlink_time; /* link keep alive time */ u32 llc_termination_rsn; /* rsn code for termination */ u8 nexthop_mac[ETH_ALEN]; u8 uses_gateway; __be32 saddr; /* net namespace */ struct net *net; u8 max_conns; /* max conn can be assigned to lgr */ u8 max_links; /* max links can be added in lgr */ u16 max_send_wr; /* number of WR buffers on send */ u16 max_recv_wr; /* number of WR buffers on recv */ }; struct { /* SMC-D */ struct smcd_gid peer_gid; /* Peer GID (remote) */ struct smcd_dev *smcd; /* ISM device for VLAN reg. */ u8 peer_shutdown : 1; /* peer triggered shutdownn */ }; }; }; struct smc_clc_msg_local; #define GID_LIST_SIZE 2 struct smc_gidlist { u8 len; u8 list[GID_LIST_SIZE][SMC_GID_SIZE]; }; struct smc_init_info_smcrv2 { /* Input fields */ __be32 saddr; struct sock *clc_sk; __be32 daddr; /* Output fields when saddr is set */ struct smc_ib_device *ib_dev_v2; u8 ib_port_v2; u8 ib_gid_v2[SMC_GID_SIZE]; /* Additional output fields when clc_sk and daddr is set as well */ u8 uses_gateway; u8 nexthop_mac[ETH_ALEN]; struct smc_gidlist gidlist; }; #define SMC_MAX_V2_ISM_DEVS SMCD_CLC_MAX_V2_GID_ENTRIES /* max # of proposed non-native ISM devices, * which can't exceed the max # of CHID-GID * entries in CLC proposal SMC-Dv2 extension. */ struct smc_init_info { u8 is_smcd; u8 smc_type_v1; u8 smc_type_v2; u8 release_nr; u8 max_conns; u8 max_links; u8 first_contact_peer; u8 first_contact_local; u16 feature_mask; unsigned short vlan_id; u32 rc; u8 negotiated_eid[SMC_MAX_EID_LEN]; /* SMC-R */ u8 smcr_version; u8 check_smcrv2; u8 peer_gid[SMC_GID_SIZE]; u8 peer_mac[ETH_ALEN]; u8 peer_systemid[SMC_SYSTEMID_LEN]; struct smc_ib_device *ib_dev; u8 ib_gid[SMC_GID_SIZE]; u8 ib_port; u32 ib_clcqpn; struct smc_init_info_smcrv2 smcrv2; /* SMC-D */ struct smcd_gid ism_peer_gid[SMC_MAX_V2_ISM_DEVS + 1]; struct smcd_dev *ism_dev[SMC_MAX_V2_ISM_DEVS + 1]; u16 ism_chid[SMC_MAX_V2_ISM_DEVS + 1]; u8 ism_offered_cnt; /* # of ISM devices offered */ u8 ism_selected; /* index of selected ISM dev*/ u8 smcd_version; }; /* Find the connection associated with the given alert token in the link group. * To use rbtrees we have to implement our own search core. * Requires @conns_lock * @token alert token to search for * @lgr link group to search in * Returns connection associated with token if found, NULL otherwise. */ static inline struct smc_connection *smc_lgr_find_conn( u32 token, struct smc_link_group *lgr) { struct smc_connection *res = NULL; struct rb_node *node; node = lgr->conns_all.rb_node; while (node) { struct smc_connection *cur = rb_entry(node, struct smc_connection, alert_node); if (cur->alert_token_local > token) { node = node->rb_left; } else { if (cur->alert_token_local < token) { node = node->rb_right; } else { res = cur; break; } } } return res; } static inline bool smc_conn_lgr_valid(struct smc_connection *conn) { return conn->lgr && conn->alert_token_local; } /* * Returns true if the specified link is usable. * * usable means the link is ready to receive RDMA messages, map memory * on the link, etc. This doesn't ensure we are able to send RDMA messages * on this link, if sending RDMA messages is needed, use smc_link_sendable() */ static inline bool smc_link_usable(struct smc_link *lnk) { if (lnk->state == SMC_LNK_UNUSED || lnk->state == SMC_LNK_INACTIVE) return false; return true; } /* * Returns true if the specified link is ready to receive AND send RDMA * messages. * * For the client side in first contact, the underlying QP may still in * RESET or RTR when the link state is ACTIVATING, checks in smc_link_usable() * is not strong enough. For those places that need to send any CDC or LLC * messages, use smc_link_sendable(), otherwise, use smc_link_usable() instead */ static inline bool smc_link_sendable(struct smc_link *lnk) { return smc_link_usable(lnk) && lnk->qp_attr.cur_qp_state == IB_QPS_RTS; } static inline bool smc_link_active(struct smc_link *lnk) { return lnk->state == SMC_LNK_ACTIVE; } static inline bool smc_link_shared_v2_rxbuf(struct smc_link *lnk) { return lnk->wr_rx_sge_cnt > 1; } static inline void smc_gid_be16_convert(__u8 *buf, u8 *gid_raw) { sprintf(buf, "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x", be16_to_cpu(((__be16 *)gid_raw)[0]), be16_to_cpu(((__be16 *)gid_raw)[1]), be16_to_cpu(((__be16 *)gid_raw)[2]), be16_to_cpu(((__be16 *)gid_raw)[3]), be16_to_cpu(((__be16 *)gid_raw)[4]), be16_to_cpu(((__be16 *)gid_raw)[5]), be16_to_cpu(((__be16 *)gid_raw)[6]), be16_to_cpu(((__be16 *)gid_raw)[7])); } struct smc_pci_dev { __u32 pci_fid; __u16 pci_pchid; __u16 pci_vendor; __u16 pci_device; __u8 pci_id[SMC_PCI_ID_STR_LEN]; }; static inline void smc_set_pci_values(struct pci_dev *pci_dev, struct smc_pci_dev *smc_dev) { smc_dev->pci_vendor = pci_dev->vendor; smc_dev->pci_device = pci_dev->device; snprintf(smc_dev->pci_id, sizeof(smc_dev->pci_id), "%s", pci_name(pci_dev)); #if IS_ENABLED(CONFIG_S390) { /* Set s390 specific PCI information */ struct zpci_dev *zdev; zdev = to_zpci(pci_dev); smc_dev->pci_fid = zdev->fid; smc_dev->pci_pchid = zdev->pchid; } #endif } struct smc_sock; struct smc_clc_msg_accept_confirm; void smc_lgr_cleanup_early(struct smc_link_group *lgr); void smc_lgr_terminate_sched(struct smc_link_group *lgr); void smc_lgr_hold(struct smc_link_group *lgr); void smc_lgr_put(struct smc_link_group *lgr); void smcr_port_add(struct smc_ib_device *smcibdev, u8 ibport); void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport); void smc_smcd_terminate(struct smcd_dev *dev, struct smcd_gid *peer_gid, unsigned short vlan); void smc_smcd_terminate_all(struct smcd_dev *dev); void smc_smcr_terminate_all(struct smc_ib_device *smcibdev); int smc_buf_create(struct smc_sock *smc, bool is_smcd); int smcd_buf_attach(struct smc_sock *smc); int smc_uncompress_bufsize(u8 compressed); int smc_rmb_rtoken_handling(struct smc_connection *conn, struct smc_link *link, struct smc_clc_msg_accept_confirm *clc); int smc_rtoken_add(struct smc_link *lnk, __be64 nw_vaddr, __be32 nw_rkey); int smc_rtoken_delete(struct smc_link *lnk, __be32 nw_rkey); void smc_rtoken_set(struct smc_link_group *lgr, int link_idx, int link_idx_new, __be32 nw_rkey_known, __be64 nw_vaddr, __be32 nw_rkey); void smc_rtoken_set2(struct smc_link_group *lgr, int rtok_idx, int link_id, __be64 nw_vaddr, __be32 nw_rkey); void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn); void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn); int smc_vlan_by_tcpsk(struct socket *clcsock, struct smc_init_info *ini); void smc_conn_free(struct smc_connection *conn); int smc_conn_create(struct smc_sock *smc, struct smc_init_info *ini); int smc_core_init(void); void smc_core_exit(void); int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk, u8 link_idx, struct smc_init_info *ini); void smcr_link_clear(struct smc_link *lnk, bool log); void smcr_link_hold(struct smc_link *lnk); void smcr_link_put(struct smc_link *lnk); void smc_switch_link_and_count(struct smc_connection *conn, struct smc_link *to_lnk); int smcr_buf_map_lgr(struct smc_link *lnk); int smcr_buf_reg_lgr(struct smc_link *lnk); void smcr_lgr_set_type(struct smc_link_group *lgr, enum smc_lgr_type new_type); void smcr_lgr_set_type_asym(struct smc_link_group *lgr, enum smc_lgr_type new_type, int asym_lnk_idx); int smcr_link_reg_buf(struct smc_link *link, struct smc_buf_desc *rmb_desc); struct smc_link *smc_switch_conns(struct smc_link_group *lgr, struct smc_link *from_lnk, bool is_dev_err); void smcr_link_down_cond(struct smc_link *lnk); void smcr_link_down_cond_sched(struct smc_link *lnk); int smc_nl_get_sys_info(struct sk_buff *skb, struct netlink_callback *cb); int smcr_nl_get_lgr(struct sk_buff *skb, struct netlink_callback *cb); int smcr_nl_get_link(struct sk_buff *skb, struct netlink_callback *cb); int smcd_nl_get_lgr(struct sk_buff *skb, struct netlink_callback *cb); static inline struct smc_link_group *smc_get_lgr(struct smc_link *link) { return link->lgr; } #endif |
| 5 5 5 5 5 5 5 5 5 4 5 4 3 3 1 1 5 4 5 1 8 6 1 6 6 7 6 5 6 2 1 1 1 1 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 | // SPDX-License-Identifier: GPL-2.0+ /* * This file contains the functions which manage clocksource drivers. * * Copyright (C) 2004, 2005 IBM, John Stultz (johnstul@us.ibm.com) */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/clocksource.h> #include <linux/cpu.h> #include <linux/delay.h> #include <linux/device.h> #include <linux/init.h> #include <linux/kthread.h> #include <linux/module.h> #include <linux/prandom.h> #include <linux/sched.h> #include <linux/tick.h> #include <linux/topology.h> #include "tick-internal.h" #include "timekeeping_internal.h" static void clocksource_enqueue(struct clocksource *cs); static noinline u64 cycles_to_nsec_safe(struct clocksource *cs, u64 start, u64 end) { u64 delta = clocksource_delta(end, start, cs->mask, cs->max_raw_delta); if (likely(delta < cs->max_cycles)) return clocksource_cyc2ns(delta, cs->mult, cs->shift); return mul_u64_u32_shr(delta, cs->mult, cs->shift); } /** * clocks_calc_mult_shift - calculate mult/shift factors for scaled math of clocks * @mult: pointer to mult variable * @shift: pointer to shift variable * @from: frequency to convert from * @to: frequency to convert to * @maxsec: guaranteed runtime conversion range in seconds * * The function evaluates the shift/mult pair for the scaled math * operations of clocksources and clockevents. * * @to and @from are frequency values in HZ. For clock sources @to is * NSEC_PER_SEC == 1GHz and @from is the counter frequency. For clock * event @to is the counter frequency and @from is NSEC_PER_SEC. * * The @maxsec conversion range argument controls the time frame in * seconds which must be covered by the runtime conversion with the * calculated mult and shift factors. This guarantees that no 64bit * overflow happens when the input value of the conversion is * multiplied with the calculated mult factor. Larger ranges may * reduce the conversion accuracy by choosing smaller mult and shift * factors. */ void clocks_calc_mult_shift(u32 *mult, u32 *shift, u32 from, u32 to, u32 maxsec) { u64 tmp; u32 sft, sftacc= 32; /* * Calculate the shift factor which is limiting the conversion * range: */ tmp = ((u64)maxsec * from) >> 32; while (tmp) { tmp >>=1; sftacc--; } /* * Find the conversion shift/mult pair which has the best * accuracy and fits the maxsec conversion range: */ for (sft = 32; sft > 0; sft--) { tmp = (u64) to << sft; tmp += from / 2; do_div(tmp, from); if ((tmp >> sftacc) == 0) break; } *mult = tmp; *shift = sft; } EXPORT_SYMBOL_GPL(clocks_calc_mult_shift); /*[Clocksource internal variables]--------- * curr_clocksource: * currently selected clocksource. * suspend_clocksource: * used to calculate the suspend time. * clocksource_list: * linked list with the registered clocksources * clocksource_mutex: * protects manipulations to curr_clocksource and the clocksource_list * override_name: * Name of the user-specified clocksource. */ static struct clocksource *curr_clocksource; static struct clocksource *suspend_clocksource; static LIST_HEAD(clocksource_list); static DEFINE_MUTEX(clocksource_mutex); static char override_name[CS_NAME_LEN]; static int finished_booting; static u64 suspend_start; #ifdef CONFIG_CLOCKSOURCE_WATCHDOG static void clocksource_watchdog_work(struct work_struct *work); static void clocksource_select(void); static LIST_HEAD(watchdog_list); static struct clocksource *watchdog; static struct timer_list watchdog_timer; static DECLARE_WORK(watchdog_work, clocksource_watchdog_work); static DEFINE_SPINLOCK(watchdog_lock); static int watchdog_running; static atomic_t watchdog_reset_pending; /* Watchdog interval: 0.5sec. */ #define WATCHDOG_INTERVAL (HZ >> 1) #define WATCHDOG_INTERVAL_NS (WATCHDOG_INTERVAL * (NSEC_PER_SEC / HZ)) /* Maximum time between two reference watchdog readouts */ #define WATCHDOG_READOUT_MAX_NS (50U * NSEC_PER_USEC) /* * Maximum time between two remote readouts for NUMA=n. On NUMA enabled systems * the timeout is calculated from the numa distance. */ #define WATCHDOG_DEFAULT_TIMEOUT_NS (50U * NSEC_PER_USEC) /* * Remote timeout NUMA distance multiplier. The local distance is 10. The * default remote distance is 20. ACPI tables provide more accurate numbers * which are guaranteed to be greater than the local distance. * * This results in a 5us base value, which is equivalent to the above !NUMA * default. */ #define WATCHDOG_NUMA_MULTIPLIER_NS ((u64)(WATCHDOG_DEFAULT_TIMEOUT_NS / LOCAL_DISTANCE)) /* Limit the NUMA timeout in case the distance values are insanely big */ #define WATCHDOG_NUMA_MAX_TIMEOUT_NS ((u64)(500U * NSEC_PER_USEC)) /* Shift values to calculate the approximate $N ppm of a given delta. */ #define SHIFT_500PPM 11 #define SHIFT_4000PPM 8 /* Number of attempts to read the watchdog */ #define WATCHDOG_FREQ_RETRIES 3 /* Five reads local and remote for inter CPU skew detection */ #define WATCHDOG_REMOTE_MAX_SEQ 10 static inline void clocksource_watchdog_lock(unsigned long *flags) { spin_lock_irqsave(&watchdog_lock, *flags); } static inline void clocksource_watchdog_unlock(unsigned long *flags) { spin_unlock_irqrestore(&watchdog_lock, *flags); } static int clocksource_watchdog_kthread(void *data); static void clocksource_watchdog_work(struct work_struct *work) { /* * We cannot directly run clocksource_watchdog_kthread() here, because * clocksource_select() calls timekeeping_notify() which uses * stop_machine(). One cannot use stop_machine() from a workqueue() due * lock inversions wrt CPU hotplug. * * Also, we only ever run this work once or twice during the lifetime * of the kernel, so there is no point in creating a more permanent * kthread for this. * * If kthread_run fails the next watchdog scan over the * watchdog_list will find the unstable clock again. */ kthread_run(clocksource_watchdog_kthread, NULL, "kwatchdog"); } static void clocksource_change_rating(struct clocksource *cs, int rating) { list_del(&cs->list); cs->rating = rating; clocksource_enqueue(cs); } static void __clocksource_unstable(struct clocksource *cs) { cs->flags &= ~(CLOCK_SOURCE_VALID_FOR_HRES | CLOCK_SOURCE_WATCHDOG); cs->flags |= CLOCK_SOURCE_UNSTABLE; /* * If the clocksource is registered clocksource_watchdog_kthread() will * re-rate and re-select. */ if (list_empty(&cs->list)) { cs->rating = 0; return; } if (cs->mark_unstable) cs->mark_unstable(cs); /* kick clocksource_watchdog_kthread() */ if (finished_booting) schedule_work(&watchdog_work); } /** * clocksource_mark_unstable - mark clocksource unstable via watchdog * @cs: clocksource to be marked unstable * * This function is called by the x86 TSC code to mark clocksources as unstable; * it defers demotion and re-selection to a kthread. */ void clocksource_mark_unstable(struct clocksource *cs) { unsigned long flags; spin_lock_irqsave(&watchdog_lock, flags); if (!(cs->flags & CLOCK_SOURCE_UNSTABLE)) { if (!list_empty(&cs->list) && list_empty(&cs->wd_list)) list_add(&cs->wd_list, &watchdog_list); __clocksource_unstable(cs); } spin_unlock_irqrestore(&watchdog_lock, flags); } static inline void clocksource_reset_watchdog(void) { struct clocksource *cs; list_for_each_entry(cs, &watchdog_list, wd_list) cs->flags &= ~CLOCK_SOURCE_WATCHDOG; } enum wd_result { WD_SUCCESS, WD_FREQ_NO_WATCHDOG, WD_FREQ_TIMEOUT, WD_FREQ_RESET, WD_FREQ_SKEWED, WD_CPU_TIMEOUT, WD_CPU_SKEWED, }; struct watchdog_cpu_data { /* Keep first as it is 32 byte aligned */ call_single_data_t csd; atomic_t remote_inprogress; enum wd_result result; u64 cpu_ts[2]; struct clocksource *cs; /* Ensure that the sequence is in a separate cache line */ atomic_t seq ____cacheline_aligned; /* Set by the control CPU according to NUMA distance */ u64 timeout_ns; }; struct watchdog_data { raw_spinlock_t lock; enum wd_result result; u64 wd_seq; u64 wd_delta; u64 cs_delta; u64 cpu_ts[2]; unsigned int curr_cpu; } ____cacheline_aligned_in_smp; static void watchdog_check_skew_remote(void *unused); static DEFINE_PER_CPU_ALIGNED(struct watchdog_cpu_data, watchdog_cpu_data) = { .csd = CSD_INIT(watchdog_check_skew_remote, NULL), }; static struct watchdog_data watchdog_data = { .lock = __RAW_SPIN_LOCK_UNLOCKED(watchdog_data.lock), }; static inline void watchdog_set_result(struct watchdog_cpu_data *wd, enum wd_result result) { guard(raw_spinlock)(&watchdog_data.lock); if (!wd->result) { atomic_set(&wd->seq, WATCHDOG_REMOTE_MAX_SEQ); WRITE_ONCE(wd->result, result); } } /* Wait for the sequence number to hand over control. */ static bool watchdog_wait_seq(struct watchdog_cpu_data *wd, u64 start, int seq) { for(int cnt = 0; atomic_read(&wd->seq) < seq; cnt++) { /* Bail if the other side set an error result */ if (READ_ONCE(wd->result) != WD_SUCCESS) return false; /* Prevent endless loops if the other CPU does not react. */ if (cnt == 5000) { u64 nsecs = ktime_get_raw_fast_ns(); if (nsecs - start >=wd->timeout_ns) { watchdog_set_result(wd, WD_CPU_TIMEOUT); return false; } cnt = 0; } cpu_relax(); } return seq < WATCHDOG_REMOTE_MAX_SEQ; } static void watchdog_check_skew(struct watchdog_cpu_data *wd, int index) { u64 prev, now, delta, start = ktime_get_raw_fast_ns(); int local = index, remote = (index + 1) & 0x1; struct clocksource *cs = wd->cs; /* Set the local timestamp so that the first iteration works correctly */ wd->cpu_ts[local] = cs->read(cs); /* Signal arrival */ atomic_inc(&wd->seq); for (int seq = local + 2; seq < WATCHDOG_REMOTE_MAX_SEQ; seq += 2) { if (!watchdog_wait_seq(wd, start, seq)) return; /* Capture local timestamp before possible non-local coherency overhead */ now = cs->read(cs); /* Store local timestamp before reading remote to limit coherency stalls */ wd->cpu_ts[local] = now; prev = wd->cpu_ts[remote]; delta = (now - prev) & cs->mask; if (delta > cs->max_raw_delta) { watchdog_set_result(wd, WD_CPU_SKEWED); return; } /* Hand over to the remote CPU */ atomic_inc(&wd->seq); } } static void watchdog_check_skew_remote(void *unused) { struct watchdog_cpu_data *wd = this_cpu_ptr(&watchdog_cpu_data); atomic_inc(&wd->remote_inprogress); watchdog_check_skew(wd, 1); atomic_dec(&wd->remote_inprogress); } static inline bool wd_csd_locked(struct watchdog_cpu_data *wd) { return READ_ONCE(wd->csd.node.u_flags) & CSD_FLAG_LOCK; } /* * This is only invoked for remote CPUs. See watchdog_check_cpu_skew(). */ static inline u64 wd_get_remote_timeout(unsigned int remote_cpu) { unsigned int n1, n2; u64 ns; if (nr_node_ids == 1) return WATCHDOG_DEFAULT_TIMEOUT_NS; n1 = cpu_to_node(smp_processor_id()); n2 = cpu_to_node(remote_cpu); ns = WATCHDOG_NUMA_MULTIPLIER_NS * node_distance(n1, n2); return min(ns, WATCHDOG_NUMA_MAX_TIMEOUT_NS); } static void __watchdog_check_cpu_skew(struct clocksource *cs, unsigned int cpu) { struct watchdog_cpu_data *wd; wd = per_cpu_ptr(&watchdog_cpu_data, cpu); if (atomic_read(&wd->remote_inprogress) || wd_csd_locked(wd)) { watchdog_data.result = WD_CPU_TIMEOUT; return; } atomic_set(&wd->seq, 0); wd->result = WD_SUCCESS; wd->cs = cs; /* Store the current CPU ID for the watchdog test unit */ cs->wd_cpu = smp_processor_id(); wd->timeout_ns = wd_get_remote_timeout(cpu); /* Kick the remote CPU into the watchdog function */ if (WARN_ON_ONCE(smp_call_function_single_async(cpu, &wd->csd))) { watchdog_data.result = WD_CPU_TIMEOUT; return; } scoped_guard(irq) watchdog_check_skew(wd, 0); scoped_guard(raw_spinlock_irq, &watchdog_data.lock) { watchdog_data.result = wd->result; memcpy(watchdog_data.cpu_ts, wd->cpu_ts, sizeof(wd->cpu_ts)); } } static void watchdog_check_cpu_skew(struct clocksource *cs) { unsigned int cpu = watchdog_data.curr_cpu; cpu = cpumask_next_wrap(cpu, cpu_online_mask); watchdog_data.curr_cpu = cpu; /* Skip the current CPU. Handles num_online_cpus() == 1 as well */ if (cpu == smp_processor_id()) return; /* Don't interfere with the test mechanics */ if ((cs->flags & CLOCK_SOURCE_WDTEST) && !(cs->flags & CLOCK_SOURCE_WDTEST_PERCPU)) return; __watchdog_check_cpu_skew(cs, cpu); } static bool watchdog_check_freq(struct clocksource *cs, bool reset_pending) { unsigned int ppm_shift = SHIFT_4000PPM; u64 wd_ts0, wd_ts1, cs_ts; watchdog_data.result = WD_SUCCESS; if (!watchdog) { watchdog_data.result = WD_FREQ_NO_WATCHDOG; return false; } if (cs->flags & CLOCK_SOURCE_WDTEST_PERCPU) return true; /* * If both the clocksource and the watchdog claim they are * calibrated use 500ppm limit. Uncalibrated clocksources need a * larger allowance because thefirmware supplied frequencies can be * way off. */ if (watchdog->flags & CLOCK_SOURCE_CALIBRATED && cs->flags & CLOCK_SOURCE_CALIBRATED) ppm_shift = SHIFT_500PPM; for (int retries = 0; retries < WATCHDOG_FREQ_RETRIES; retries++) { s64 wd_last, cs_last, wd_seq, wd_delta, cs_delta, max_delta; scoped_guard(irq) { wd_ts0 = watchdog->read(watchdog); cs_ts = cs->read(cs); wd_ts1 = watchdog->read(watchdog); } wd_last = cs->wd_last; cs_last = cs->cs_last; /* Validate the watchdog readout window */ wd_seq = cycles_to_nsec_safe(watchdog, wd_ts0, wd_ts1); if (wd_seq > WATCHDOG_READOUT_MAX_NS) { /* Store for printout in case all retries fail */ watchdog_data.wd_seq = wd_seq; continue; } /* Store for subsequent processing */ cs->wd_last = wd_ts0; cs->cs_last = cs_ts; /* First round or reset pending? */ if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) || reset_pending) goto reset; /* Calculate the nanosecond deltas from the last invocation */ wd_delta = cycles_to_nsec_safe(watchdog, wd_last, wd_ts0); cs_delta = cycles_to_nsec_safe(cs, cs_last, cs_ts); watchdog_data.wd_delta = wd_delta; watchdog_data.cs_delta = cs_delta; /* * Ensure that the deltas are within the readout limits of * the clocksource and the watchdog. Long delays can cause * clocksources to overflow. */ max_delta = max(wd_delta, cs_delta); if (max_delta > cs->max_idle_ns || max_delta > watchdog->max_idle_ns) goto reset; /* * Calculate and validate the skew against the allowed PPM * value of the maximum delta plus the watchdog readout * time. */ if (abs(wd_delta - cs_delta) < (max_delta >> ppm_shift) + wd_seq) return true; watchdog_data.result = WD_FREQ_SKEWED; return false; } watchdog_data.result = WD_FREQ_TIMEOUT; return false; reset: cs->flags |= CLOCK_SOURCE_WATCHDOG; watchdog_data.result = WD_FREQ_RESET; return false; } /* Synchronization for sched clock */ static void clocksource_tick_stable(struct clocksource *cs) { if (cs == curr_clocksource && cs->tick_stable) cs->tick_stable(cs); } /* Conditionaly enable high resolution mode */ static void clocksource_enable_highres(struct clocksource *cs) { if ((cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) || !(cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) || !watchdog || !(watchdog->flags & CLOCK_SOURCE_IS_CONTINUOUS)) return; /* Mark it valid for high-res. */ cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES; /* * Can't schedule work before finished_booting is * true. clocksource_done_booting will take care of it. */ if (!finished_booting) return; if (cs->flags & CLOCK_SOURCE_WDTEST) return; /* * If this is not the current clocksource let the watchdog thread * reselect it. Due to the change to high res this clocksource * might be preferred now. If it is the current clocksource let the * tick code know about that change. */ if (cs != curr_clocksource) { cs->flags |= CLOCK_SOURCE_RESELECT; schedule_work(&watchdog_work); } else { tick_clock_notify(); } } static DEFINE_RATELIMIT_STATE(ratelimit_state, 5 * HZ, 2); static void watchdog_print_freq_timeout(struct clocksource *cs) { if (!__ratelimit(&ratelimit_state)) return; pr_info("Watchdog %s read timed out. Readout sequence took: %lluns\n", watchdog->name, watchdog_data.wd_seq); } static void watchdog_print_freq_skew(struct clocksource *cs) { pr_warn("Marking clocksource %s unstable due to frequency skew\n", cs->name); pr_warn("Watchdog %20s interval: %16lluns\n", watchdog->name, watchdog_data.wd_delta); pr_warn("Clocksource %20s interval: %16lluns\n", cs->name, watchdog_data.cs_delta); } static void watchdog_handle_remote_timeout(struct clocksource *cs) { pr_info_once("Watchdog remote CPU %u read timed out\n", watchdog_data.curr_cpu); } static void watchdog_print_remote_skew(struct clocksource *cs) { pr_warn("Marking clocksource %s unstable due to inter CPU skew\n", cs->name); if (watchdog_data.cpu_ts[0] < watchdog_data.cpu_ts[1]) { pr_warn("CPU%u %16llu < CPU%u %16llu (cycles)\n", smp_processor_id(), watchdog_data.cpu_ts[0], watchdog_data.curr_cpu, watchdog_data.cpu_ts[1]); } else { pr_warn("CPU%u %16llu < CPU%u %16llu (cycles)\n", watchdog_data.curr_cpu, watchdog_data.cpu_ts[1], smp_processor_id(), watchdog_data.cpu_ts[0]); } } static void watchdog_check_result(struct clocksource *cs) { switch (watchdog_data.result) { case WD_SUCCESS: clocksource_tick_stable(cs); clocksource_enable_highres(cs); return; case WD_FREQ_TIMEOUT: watchdog_print_freq_timeout(cs); /* Try again later and invalidate the reference timestamps. */ cs->flags &= ~CLOCK_SOURCE_WATCHDOG; return; case WD_FREQ_NO_WATCHDOG: case WD_FREQ_RESET: /* * Nothing to do when the reference timestamps were reset * or no watchdog clocksource registered. */ return; case WD_FREQ_SKEWED: watchdog_print_freq_skew(cs); break; case WD_CPU_TIMEOUT: /* Remote check timed out. Try again next cycle. */ watchdog_handle_remote_timeout(cs); return; case WD_CPU_SKEWED: watchdog_print_remote_skew(cs); break; } __clocksource_unstable(cs); } static void clocksource_watchdog(struct timer_list *unused) { struct clocksource *cs; bool reset_pending; guard(spinlock)(&watchdog_lock); if (!watchdog_running) return; reset_pending = atomic_read(&watchdog_reset_pending); list_for_each_entry(cs, &watchdog_list, wd_list) { /* Clocksource already marked unstable? */ if (cs->flags & CLOCK_SOURCE_UNSTABLE) { if (finished_booting) schedule_work(&watchdog_work); continue; } /* Compare against watchdog clocksource if available */ if (watchdog_check_freq(cs, reset_pending)) { /* Check for inter CPU skew */ watchdog_check_cpu_skew(cs); } watchdog_check_result(cs); } /* Clear after the full clocksource walk */ if (reset_pending) atomic_dec(&watchdog_reset_pending); /* Could have been rearmed by a stop/start cycle */ if (!timer_pending(&watchdog_timer)) { watchdog_timer.expires += WATCHDOG_INTERVAL; add_timer_local(&watchdog_timer); } } static inline void clocksource_start_watchdog(void) { if (watchdog_running || list_empty(&watchdog_list)) return; timer_setup(&watchdog_timer, clocksource_watchdog, TIMER_PINNED); watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL; add_timer_on(&watchdog_timer, get_boot_cpu_id()); watchdog_running = 1; } static inline void clocksource_stop_watchdog(void) { if (!watchdog_running || !list_empty(&watchdog_list)) return; timer_delete(&watchdog_timer); watchdog_running = 0; } static void clocksource_resume_watchdog(void) { atomic_inc(&watchdog_reset_pending); } static void clocksource_enqueue_watchdog(struct clocksource *cs) { INIT_LIST_HEAD(&cs->wd_list); if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) { /* cs is a clocksource to be watched. */ list_add(&cs->wd_list, &watchdog_list); cs->flags &= ~CLOCK_SOURCE_WATCHDOG; } else { /* cs is a watchdog. */ if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES; } } static void clocksource_select_watchdog(bool fallback) { struct clocksource *cs, *old_wd; unsigned long flags; spin_lock_irqsave(&watchdog_lock, flags); /* save current watchdog */ old_wd = watchdog; if (fallback) watchdog = NULL; list_for_each_entry(cs, &clocksource_list, list) { /* cs is a clocksource to be watched. */ if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) continue; /* * If it's not continuous, don't put the fox in charge of * the henhouse. */ if (!(cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)) continue; /* Skip current if we were requested for a fallback. */ if (fallback && cs == old_wd) continue; /* Pick the best watchdog. */ if (!watchdog || cs->rating > watchdog->rating) watchdog = cs; } /* If we failed to find a fallback restore the old one. */ if (!watchdog) watchdog = old_wd; /* If we changed the watchdog we need to reset cycles. */ if (watchdog != old_wd) clocksource_reset_watchdog(); /* Check if the watchdog timer needs to be started. */ clocksource_start_watchdog(); spin_unlock_irqrestore(&watchdog_lock, flags); } static void clocksource_dequeue_watchdog(struct clocksource *cs) { if (cs != watchdog) { if (cs->flags & CLOCK_SOURCE_MUST_VERIFY) { /* cs is a watched clocksource. */ list_del_init(&cs->wd_list); /* Check if the watchdog timer needs to be stopped. */ clocksource_stop_watchdog(); } } } static int __clocksource_watchdog_kthread(void) { struct clocksource *cs, *tmp; unsigned long flags; int select = 0; spin_lock_irqsave(&watchdog_lock, flags); list_for_each_entry_safe(cs, tmp, &watchdog_list, wd_list) { if (cs->flags & CLOCK_SOURCE_UNSTABLE) { list_del_init(&cs->wd_list); clocksource_change_rating(cs, 0); select = 1; } if (cs->flags & CLOCK_SOURCE_RESELECT) { cs->flags &= ~CLOCK_SOURCE_RESELECT; select = 1; } } /* Check if the watchdog timer needs to be stopped. */ clocksource_stop_watchdog(); spin_unlock_irqrestore(&watchdog_lock, flags); return select; } static int clocksource_watchdog_kthread(void *data) { mutex_lock(&clocksource_mutex); if (__clocksource_watchdog_kthread()) clocksource_select(); mutex_unlock(&clocksource_mutex); return 0; } static bool clocksource_is_watchdog(struct clocksource *cs) { return cs == watchdog; } #else /* CONFIG_CLOCKSOURCE_WATCHDOG */ static void clocksource_enqueue_watchdog(struct clocksource *cs) { if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) cs->flags |= CLOCK_SOURCE_VALID_FOR_HRES; } static void clocksource_select_watchdog(bool fallback) { } static inline void clocksource_dequeue_watchdog(struct clocksource *cs) { } static inline void clocksource_resume_watchdog(void) { } static inline int __clocksource_watchdog_kthread(void) { return 0; } static bool clocksource_is_watchdog(struct clocksource *cs) { return false; } void clocksource_mark_unstable(struct clocksource *cs) { } static inline void clocksource_watchdog_lock(unsigned long *flags) { } static inline void clocksource_watchdog_unlock(unsigned long *flags) { } #endif /* CONFIG_CLOCKSOURCE_WATCHDOG */ static bool clocksource_is_suspend(struct clocksource *cs) { return cs == suspend_clocksource; } static void __clocksource_suspend_select(struct clocksource *cs) { /* * Skip the clocksource which will be stopped in suspend state. */ if (!(cs->flags & CLOCK_SOURCE_SUSPEND_NONSTOP)) return; /* * The nonstop clocksource can be selected as the suspend clocksource to * calculate the suspend time, so it should not supply suspend/resume * interfaces to suspend the nonstop clocksource when system suspends. */ if (cs->suspend || cs->resume) { pr_warn("Nonstop clocksource %s should not supply suspend/resume interfaces\n", cs->name); } /* Pick the best rating. */ if (!suspend_clocksource || cs->rating > suspend_clocksource->rating) suspend_clocksource = cs; } /** * clocksource_suspend_select - Select the best clocksource for suspend timing * @fallback: if select a fallback clocksource */ static void clocksource_suspend_select(bool fallback) { struct clocksource *cs, *old_suspend; old_suspend = suspend_clocksource; if (fallback) suspend_clocksource = NULL; list_for_each_entry(cs, &clocksource_list, list) { /* Skip current if we were requested for a fallback. */ if (fallback && cs == old_suspend) continue; __clocksource_suspend_select(cs); } } /** * clocksource_start_suspend_timing - Start measuring the suspend timing * @cs: current clocksource from timekeeping * @start_cycles: current cycles from timekeeping * * This function will save the start cycle values of suspend timer to calculate * the suspend time when resuming system. * * This function is called late in the suspend process from timekeeping_suspend(), * that means processes are frozen, non-boot cpus and interrupts are disabled * now. It is therefore possible to start the suspend timer without taking the * clocksource mutex. */ void clocksource_start_suspend_timing(struct clocksource *cs, u64 start_cycles) { if (!suspend_clocksource) return; /* * If current clocksource is the suspend timer, we should use the * tkr_mono.cycle_last value as suspend_start to avoid same reading * from suspend timer. */ if (clocksource_is_suspend(cs)) { suspend_start = start_cycles; return; } if (suspend_clocksource->enable && suspend_clocksource->enable(suspend_clocksource)) { pr_warn_once("Failed to enable the non-suspend-able clocksource.\n"); return; } suspend_start = suspend_clocksource->read(suspend_clocksource); } /** * clocksource_stop_suspend_timing - Stop measuring the suspend timing * @cs: current clocksource from timekeeping * @cycle_now: current cycles from timekeeping * * This function will calculate the suspend time from suspend timer. * * Returns nanoseconds since suspend started, 0 if no usable suspend clocksource. * * This function is called early in the resume process from timekeeping_resume(), * that means there is only one cpu, no processes are running and the interrupts * are disabled. It is therefore possible to stop the suspend timer without * taking the clocksource mutex. */ u64 clocksource_stop_suspend_timing(struct clocksource *cs, u64 cycle_now) { u64 now, nsec = 0; if (!suspend_clocksource) return 0; /* * If current clocksource is the suspend timer, we should use the * tkr_mono.cycle_last value from timekeeping as current cycle to * avoid same reading from suspend timer. */ if (clocksource_is_suspend(cs)) now = cycle_now; else now = suspend_clocksource->read(suspend_clocksource); if (now > suspend_start) nsec = cycles_to_nsec_safe(suspend_clocksource, suspend_start, now); /* * Disable the suspend timer to save power if current clocksource is * not the suspend timer. */ if (!clocksource_is_suspend(cs) && suspend_clocksource->disable) suspend_clocksource->disable(suspend_clocksource); return nsec; } /** * clocksource_suspend - suspend the clocksource(s) */ void clocksource_suspend(void) { struct clocksource *cs; list_for_each_entry_reverse(cs, &clocksource_list, list) if (cs->suspend) cs->suspend(cs); } /** * clocksource_resume - resume the clocksource(s) */ void clocksource_resume(void) { struct clocksource *cs; list_for_each_entry(cs, &clocksource_list, list) if (cs->resume) cs->resume(cs); clocksource_resume_watchdog(); } /** * clocksource_touch_watchdog - Update watchdog * * Update the watchdog after exception contexts such as kgdb so as not * to incorrectly trip the watchdog. This might fail when the kernel * was stopped in code which holds watchdog_lock. */ void clocksource_touch_watchdog(void) { clocksource_resume_watchdog(); } /** * clocksource_max_adjustment- Returns max adjustment amount * @cs: Pointer to clocksource * */ static u32 clocksource_max_adjustment(struct clocksource *cs) { u64 ret; /* * We won't try to correct for more than 11% adjustments (110,000 ppm), */ ret = (u64)cs->mult * 11; do_div(ret,100); return (u32)ret; } /** * clocks_calc_max_nsecs - Returns maximum nanoseconds that can be converted * @mult: cycle to nanosecond multiplier * @shift: cycle to nanosecond divisor (power of two) * @maxadj: maximum adjustment value to mult (~11%) * @mask: bitmask for two's complement subtraction of non 64 bit counters * @max_cyc: maximum cycle value before potential overflow (does not include * any safety margin) * * NOTE: This function includes a safety margin of 50%, in other words, we * return half the number of nanoseconds the hardware counter can technically * cover. This is done so that we can potentially detect problems caused by * delayed timers or bad hardware, which might result in time intervals that * are larger than what the math used can handle without overflows. */ u64 clocks_calc_max_nsecs(u32 mult, u32 shift, u32 maxadj, u64 mask, u64 *max_cyc) { u64 max_nsecs, max_cycles; /* * Calculate the maximum number of cycles that we can pass to the * cyc2ns() function without overflowing a 64-bit result. */ max_cycles = ULLONG_MAX; do_div(max_cycles, mult+maxadj); /* * The actual maximum number of cycles we can defer the clocksource is * determined by the minimum of max_cycles and mask. * Note: Here we subtract the maxadj to make sure we don't sleep for * too long if there's a large negative adjustment. */ max_cycles = min(max_cycles, mask); max_nsecs = clocksource_cyc2ns(max_cycles, mult - maxadj, shift); /* return the max_cycles value as well if requested */ if (max_cyc) *max_cyc = max_cycles; /* Return 50% of the actual maximum, so we can detect bad values */ max_nsecs >>= 1; return max_nsecs; } /** * clocksource_update_max_deferment - Updates the clocksource max_idle_ns & max_cycles * @cs: Pointer to clocksource to be updated * */ static inline void clocksource_update_max_deferment(struct clocksource *cs) { cs->max_idle_ns = clocks_calc_max_nsecs(cs->mult, cs->shift, cs->maxadj, cs->mask, &cs->max_cycles); /* * Threshold for detecting negative motion in clocksource_delta(). * * Allow for 0.875 of the counter width so that overly long idle * sleeps, which go slightly over mask/2, do not trigger the * negative motion detection. */ cs->max_raw_delta = (cs->mask >> 1) + (cs->mask >> 2) + (cs->mask >> 3); } static struct clocksource *clocksource_find_best(bool oneshot, bool skipcur) { struct clocksource *cs; if (!finished_booting || list_empty(&clocksource_list)) return NULL; /* * We pick the clocksource with the highest rating. If oneshot * mode is active, we pick the highres valid clocksource with * the best rating. */ list_for_each_entry(cs, &clocksource_list, list) { if (skipcur && cs == curr_clocksource) continue; if (oneshot && !(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES)) continue; if (cs->flags & CLOCK_SOURCE_WDTEST) continue; return cs; } return NULL; } static void __clocksource_select(bool skipcur) { bool oneshot = tick_oneshot_mode_active(); struct clocksource *best, *cs; /* Find the best suitable clocksource */ best = clocksource_find_best(oneshot, skipcur); if (!best) return; if (!strlen(override_name)) goto found; /* Check for the override clocksource. */ list_for_each_entry(cs, &clocksource_list, list) { if (skipcur && cs == curr_clocksource) continue; if (strcmp(cs->name, override_name) != 0) continue; if (cs->flags & CLOCK_SOURCE_WDTEST) continue; /* * Check to make sure we don't switch to a non-highres * capable clocksource if the tick code is in oneshot * mode (highres or nohz) */ if (!(cs->flags & CLOCK_SOURCE_VALID_FOR_HRES) && oneshot) { /* Override clocksource cannot be used. */ if (cs->flags & CLOCK_SOURCE_UNSTABLE) { pr_warn("Override clocksource %s is unstable and not HRT compatible - cannot switch while in HRT/NOHZ mode\n", cs->name); override_name[0] = 0; } else { /* * The override cannot be currently verified. * Deferring to let the watchdog check. */ pr_info("Override clocksource %s is not currently HRT compatible - deferring\n", cs->name); } } else /* Override clocksource can be used. */ best = cs; break; } found: if (curr_clocksource != best && !timekeeping_notify(best)) { pr_info("Switched to clocksource %s\n", best->name); curr_clocksource = best; } } /** * clocksource_select - Select the best clocksource available * * Private function. Must hold clocksource_mutex when called. * * Select the clocksource with the best rating, or the clocksource, * which is selected by userspace override. */ static void clocksource_select(void) { __clocksource_select(false); } static void clocksource_select_fallback(void) { __clocksource_select(true); } /* * clocksource_done_booting - Called near the end of core bootup * * Hack to avoid lots of clocksource churn at boot time. * We use fs_initcall because we want this to start before * device_initcall but after subsys_initcall. */ static int __init clocksource_done_booting(void) { mutex_lock(&clocksource_mutex); curr_clocksource = clocksource_default_clock(); finished_booting = 1; /* * Run the watchdog first to eliminate unstable clock sources */ __clocksource_watchdog_kthread(); clocksource_select(); mutex_unlock(&clocksource_mutex); return 0; } fs_initcall(clocksource_done_booting); /* * Enqueue the clocksource sorted by rating */ static void clocksource_enqueue(struct clocksource *cs) { struct list_head *entry = &clocksource_list; struct clocksource *tmp; list_for_each_entry(tmp, &clocksource_list, list) { /* Keep track of the place, where to insert */ if (tmp->rating < cs->rating) break; entry = &tmp->list; } list_add(&cs->list, entry); } /** * __clocksource_update_freq_scale - Used update clocksource with new freq * @cs: clocksource to be registered * @scale: Scale factor multiplied against freq to get clocksource hz * @freq: clocksource frequency (cycles per second) divided by scale */ static void __clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq) { u64 sec; /* * Default clocksources are *special* and self-define their mult/shift. * But, you're not special, so you should specify a freq value. */ if (freq) { /* * Calc the maximum number of seconds which we can run before * wrapping around. For clocksources which have a mask > 32-bit * we need to limit the max sleep time to have a good * conversion precision. 10 minutes is still a reasonable * amount. That results in a shift value of 24 for a * clocksource with mask >= 40-bit and f >= 4GHz. That maps to * ~ 0.06ppm granularity for NTP. */ sec = cs->mask; do_div(sec, freq); do_div(sec, scale); if (!sec) sec = 1; else if (sec > 600 && cs->mask > UINT_MAX) sec = 600; clocks_calc_mult_shift(&cs->mult, &cs->shift, freq, NSEC_PER_SEC / scale, sec * scale); /* Update cs::freq_khz */ cs->freq_khz = div_u64((u64)freq * scale, 1000); } /* * Ensure clocksources that have large 'mult' values don't overflow * when adjusted. */ cs->maxadj = clocksource_max_adjustment(cs); while (freq && ((cs->mult + cs->maxadj < cs->mult) || (cs->mult - cs->maxadj > cs->mult))) { cs->mult >>= 1; cs->shift--; cs->maxadj = clocksource_max_adjustment(cs); } /* * Only warn for *special* clocksources that self-define * their mult/shift values and don't specify a freq. */ WARN_ONCE(cs->mult + cs->maxadj < cs->mult, "timekeeping: Clocksource %s might overflow on 11%% adjustment\n", cs->name); clocksource_update_max_deferment(cs); pr_info("%s: mask: 0x%llx max_cycles: 0x%llx, max_idle_ns: %lld ns\n", cs->name, cs->mask, cs->max_cycles, cs->max_idle_ns); } /** * __clocksource_register_scale - Used to install new clocksources * @cs: clocksource to be registered * @scale: Scale factor multiplied against freq to get clocksource hz * @freq: clocksource frequency (cycles per second) divided by scale * * Returns -EBUSY if registration fails, zero otherwise. * * This *SHOULD NOT* be called directly! Please use the * clocksource_register_hz() or clocksource_register_khz helper functions. */ int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq) { unsigned long flags; clocksource_arch_init(cs); if (WARN_ON_ONCE((unsigned int)cs->id >= CSID_MAX)) cs->id = CSID_GENERIC; if (WARN_ON_ONCE(!freq && cs->flags & CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT)) cs->flags &= ~CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT; if (cs->vdso_clock_mode < 0 || cs->vdso_clock_mode >= VDSO_CLOCKMODE_MAX) { pr_warn("clocksource %s registered with invalid VDSO mode %d. Disabling VDSO support.\n", cs->name, cs->vdso_clock_mode); cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; } /* Initialize mult/shift and max_idle_ns */ __clocksource_update_freq_scale(cs, scale, freq); /* Add clocksource to the clocksource list */ mutex_lock(&clocksource_mutex); clocksource_watchdog_lock(&flags); clocksource_enqueue(cs); clocksource_enqueue_watchdog(cs); clocksource_watchdog_unlock(&flags); clocksource_select(); clocksource_select_watchdog(false); __clocksource_suspend_select(cs); mutex_unlock(&clocksource_mutex); return 0; } EXPORT_SYMBOL_GPL(__clocksource_register_scale); static void __devm_clocksource_unregister(void *data) { struct clocksource *cs = data; clocksource_unregister(cs); } int __devm_clocksource_register_scale(struct device *dev, struct clocksource *cs, u32 scale, u32 freq) { int ret; ret = __clocksource_register_scale(cs, scale, freq); if (ret) return ret; return devm_add_action_or_reset(dev, __devm_clocksource_unregister, cs); } EXPORT_SYMBOL_GPL(__devm_clocksource_register_scale); /* * Unbind clocksource @cs. Called with clocksource_mutex held */ static int clocksource_unbind(struct clocksource *cs) { unsigned long flags; if (clocksource_is_watchdog(cs)) { /* Select and try to install a replacement watchdog. */ clocksource_select_watchdog(true); if (clocksource_is_watchdog(cs)) return -EBUSY; } if (cs == curr_clocksource) { /* Select and try to install a replacement clock source */ clocksource_select_fallback(); if (curr_clocksource == cs) return -EBUSY; } if (clocksource_is_suspend(cs)) { /* * Select and try to install a replacement suspend clocksource. * If no replacement suspend clocksource, we will just let the * clocksource go and have no suspend clocksource. */ clocksource_suspend_select(true); } clocksource_watchdog_lock(&flags); clocksource_dequeue_watchdog(cs); list_del_init(&cs->list); clocksource_watchdog_unlock(&flags); return 0; } /** * clocksource_unregister - remove a registered clocksource * @cs: clocksource to be unregistered */ int clocksource_unregister(struct clocksource *cs) { int ret = 0; mutex_lock(&clocksource_mutex); if (!list_empty(&cs->list)) ret = clocksource_unbind(cs); mutex_unlock(&clocksource_mutex); return ret; } EXPORT_SYMBOL(clocksource_unregister); #ifdef CONFIG_SYSFS /** * current_clocksource_show - sysfs interface for current clocksource * @dev: unused * @attr: unused * @buf: char buffer to be filled with clocksource list * * Provides sysfs interface for listing current clocksource. */ static ssize_t current_clocksource_show(struct device *dev, struct device_attribute *attr, char *buf) { ssize_t count = 0; mutex_lock(&clocksource_mutex); count = sysfs_emit(buf, "%s\n", curr_clocksource->name); mutex_unlock(&clocksource_mutex); return count; } ssize_t sysfs_get_uname(const char *buf, char *dst, size_t cnt) { size_t ret = cnt; /* strings from sysfs write are not 0 terminated! */ if (!cnt || cnt >= CS_NAME_LEN) return -EINVAL; /* strip of \n: */ if (buf[cnt-1] == '\n') cnt--; if (cnt > 0) memcpy(dst, buf, cnt); dst[cnt] = 0; return ret; } /** * current_clocksource_store - interface for manually overriding clocksource * @dev: unused * @attr: unused * @buf: name of override clocksource * @count: length of buffer * * Takes input from sysfs interface for manually overriding the default * clocksource selection. */ static ssize_t current_clocksource_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { ssize_t ret; mutex_lock(&clocksource_mutex); ret = sysfs_get_uname(buf, override_name, count); if (ret >= 0) clocksource_select(); mutex_unlock(&clocksource_mutex); return ret; } static DEVICE_ATTR_RW(current_clocksource); /** * unbind_clocksource_store - interface for manually unbinding clocksource * @dev: unused * @attr: unused * @buf: unused * @count: length of buffer * * Takes input from sysfs interface for manually unbinding a clocksource. */ static ssize_t unbind_clocksource_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) { struct clocksource *cs; char name[CS_NAME_LEN]; ssize_t ret; ret = sysfs_get_uname(buf, name, count); if (ret < 0) return ret; ret = -ENODEV; mutex_lock(&clocksource_mutex); list_for_each_entry(cs, &clocksource_list, list) { if (strcmp(cs->name, name)) continue; ret = clocksource_unbind(cs); break; } mutex_unlock(&clocksource_mutex); return ret ? ret : count; } static DEVICE_ATTR_WO(unbind_clocksource); /** * available_clocksource_show - sysfs interface for listing clocksource * @dev: unused * @attr: unused * @buf: char buffer to be filled with clocksource list * * Provides sysfs interface for listing registered clocksources */ static ssize_t available_clocksource_show(struct device *dev, struct device_attribute *attr, char *buf) { struct clocksource *src; ssize_t count = 0; mutex_lock(&clocksource_mutex); list_for_each_entry(src, &clocksource_list, list) { /* * Don't show non-HRES clocksource if the tick code is * in one shot mode (highres=on or nohz=on) */ if (!tick_oneshot_mode_active() || (src->flags & CLOCK_SOURCE_VALID_FOR_HRES)) count += snprintf(buf + count, max((ssize_t)PAGE_SIZE - count, (ssize_t)0), "%s ", src->name); } mutex_unlock(&clocksource_mutex); count += snprintf(buf + count, max((ssize_t)PAGE_SIZE - count, (ssize_t)0), "\n"); return count; } static DEVICE_ATTR_RO(available_clocksource); static struct attribute *clocksource_attrs[] = { &dev_attr_current_clocksource.attr, &dev_attr_unbind_clocksource.attr, &dev_attr_available_clocksource.attr, NULL }; ATTRIBUTE_GROUPS(clocksource); static const struct bus_type clocksource_subsys = { .name = "clocksource", .dev_name = "clocksource", }; static struct device device_clocksource = { .id = 0, .bus = &clocksource_subsys, .groups = clocksource_groups, }; static int __init init_clocksource_sysfs(void) { int error = subsys_system_register(&clocksource_subsys, NULL); if (!error) error = device_register(&device_clocksource); return error; } device_initcall(init_clocksource_sysfs); #endif /* CONFIG_SYSFS */ /** * boot_override_clocksource - boot clock override * @str: override name * * Takes a clocksource= boot argument and uses it * as the clocksource override name. */ static int __init boot_override_clocksource(char* str) { mutex_lock(&clocksource_mutex); if (str) strscpy(override_name, str); mutex_unlock(&clocksource_mutex); return 1; } __setup("clocksource=", boot_override_clocksource); /** * boot_override_clock - Compatibility layer for deprecated boot option * @str: override name * * DEPRECATED! Takes a clock= boot argument and uses it * as the clocksource override name */ static int __init boot_override_clock(char* str) { if (!strcmp(str, "pmtmr")) { pr_warn("clock=pmtmr is deprecated - use clocksource=acpi_pm\n"); return boot_override_clocksource("acpi_pm"); } pr_warn("clock= boot option is deprecated - use clocksource=xyz\n"); return boot_override_clocksource(str); } __setup("clock=", boot_override_clock); |
| 124 92 318 348 2766 344 4100 4953 4688 5406 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | /** * css_get - obtain a reference on the specified css * @css: target css * * The caller must already have a reference. */ CGROUP_REF_FN_ATTRS void css_get(struct cgroup_subsys_state *css) { if (!(css->flags & CSS_NO_REF)) percpu_ref_get(&css->refcnt); } CGROUP_REF_EXPORT(css_get) /** * css_get_many - obtain references on the specified css * @css: target css * @n: number of references to get * * The caller must already have a reference. */ CGROUP_REF_FN_ATTRS void css_get_many(struct cgroup_subsys_state *css, unsigned int n) { if (!(css->flags & CSS_NO_REF)) percpu_ref_get_many(&css->refcnt, n); } CGROUP_REF_EXPORT(css_get_many) /** * css_tryget - try to obtain a reference on the specified css * @css: target css * * Obtain a reference on @css unless it already has reached zero and is * being released. This function doesn't care whether @css is on or * offline. The caller naturally needs to ensure that @css is accessible * but doesn't have to be holding a reference on it - IOW, RCU protected * access is good enough for this function. Returns %true if a reference * count was successfully obtained; %false otherwise. */ CGROUP_REF_FN_ATTRS bool css_tryget(struct cgroup_subsys_state *css) { if (!(css->flags & CSS_NO_REF)) return percpu_ref_tryget(&css->refcnt); return true; } CGROUP_REF_EXPORT(css_tryget) /** * css_tryget_online - try to obtain a reference on the specified css if online * @css: target css * * Obtain a reference on @css if it's online. The caller naturally needs * to ensure that @css is accessible but doesn't have to be holding a * reference on it - IOW, RCU protected access is good enough for this * function. Returns %true if a reference count was successfully obtained; * %false otherwise. */ CGROUP_REF_FN_ATTRS bool css_tryget_online(struct cgroup_subsys_state *css) { if (!(css->flags & CSS_NO_REF)) return percpu_ref_tryget_live(&css->refcnt); return true; } CGROUP_REF_EXPORT(css_tryget_online) /** * css_put - put a css reference * @css: target css * * Put a reference obtained via css_get() and css_tryget_online(). */ CGROUP_REF_FN_ATTRS void css_put(struct cgroup_subsys_state *css) { if (!(css->flags & CSS_NO_REF)) percpu_ref_put(&css->refcnt); } CGROUP_REF_EXPORT(css_put) /** * css_put_many - put css references * @css: target css * @n: number of references to put * * Put references obtained via css_get() and css_tryget_online(). */ CGROUP_REF_FN_ATTRS void css_put_many(struct cgroup_subsys_state *css, unsigned int n) { if (!(css->flags & CSS_NO_REF)) percpu_ref_put_many(&css->refcnt, n); } CGROUP_REF_EXPORT(css_put_many) |
| 14 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 | /* SPDX-License-Identifier: GPL-2.0 */ /* * thermal_core.h * * Copyright (C) 2012 Intel Corp * Author: Durgadoss R <durgadoss.r@intel.com> */ #ifndef __THERMAL_CORE_H__ #define __THERMAL_CORE_H__ #include <linux/cleanup.h> #include <linux/device.h> #include <linux/thermal.h> #include "thermal_netlink.h" #include "thermal_thresholds.h" #include "thermal_debugfs.h" struct thermal_attr { struct device_attribute attr; char name[THERMAL_NAME_LENGTH]; }; struct thermal_trip_attrs { struct thermal_attr type; struct thermal_attr temp; struct thermal_attr hyst; }; struct thermal_trip_desc { struct thermal_trip trip; struct thermal_trip_attrs trip_attrs; struct list_head list_node; struct list_head thermal_instances; int threshold; }; /** * struct thermal_governor - structure that holds thermal governor information * @name: name of the governor * @bind_to_tz: callback called when binding to a thermal zone. If it * returns 0, the governor is bound to the thermal zone, * otherwise it fails. * @unbind_from_tz: callback called when a governor is unbound from a * thermal zone. * @trip_crossed: called for trip points that have just been crossed * @manage: called on thermal zone temperature updates * @update_tz: callback called when thermal zone internals have changed, e.g. * thermal cooling instance was added/removed * @governor_list: node in thermal_governor_list (in thermal_core.c) */ struct thermal_governor { const char *name; int (*bind_to_tz)(struct thermal_zone_device *tz); void (*unbind_from_tz)(struct thermal_zone_device *tz); void (*trip_crossed)(struct thermal_zone_device *tz, const struct thermal_trip *trip, bool upward); void (*manage)(struct thermal_zone_device *tz); void (*update_tz)(struct thermal_zone_device *tz, enum thermal_notify_event reason); struct list_head governor_list; }; #define TZ_STATE_FLAG_SUSPENDED BIT(0) #define TZ_STATE_FLAG_RESUMING BIT(1) #define TZ_STATE_FLAG_INIT BIT(2) #define TZ_STATE_FLAG_EXIT BIT(3) #define TZ_STATE_READY 0 /** * struct thermal_zone_device - structure for a thermal zone * @id: unique id number for each thermal zone * @type: the thermal zone device type * @device: &struct device for this thermal zone * @removal: removal completion * @resume: resume completion * @trips_attribute_group: trip point sysfs attributes * @trips_high: trips above the current zone temperature * @trips_reached: trips below or at the current zone temperature * @trips_invalid: trips with invalid temperature * @mode: current mode of this thermal zone * @devdata: private pointer for device private data * @num_trips: number of trip points the thermal zone supports * @passive_delay_jiffies: number of jiffies to wait between polls when * performing passive cooling. * @polling_delay_jiffies: number of jiffies to wait between polls when * checking whether trip points have been crossed (0 for * interrupt driven systems) * @recheck_delay_jiffies: delay after a failed attempt to determine the zone * temperature before trying again * @temperature: current temperature. This is only for core code, * drivers should use thermal_zone_get_temp() to get the * current temperature * @last_temperature: previous temperature read * @emul_temperature: emulated temperature when using CONFIG_THERMAL_EMULATION * @passive: 1 if you've crossed a passive trip point, 0 otherwise. * @prev_low_trip: the low current temperature if you've crossed a passive * trip point. * @prev_high_trip: the above current temperature if you've crossed a * passive trip point. * @ops: operations this &thermal_zone_device supports * @tzp: thermal zone parameters * @governor: pointer to the governor for this thermal zone * @governor_data: private pointer for governor data * @ida: &struct ida to generate unique id for this zone's cooling * devices * @lock: lock to protect thermal_instances list * @node: node in thermal_tz_list (in thermal_core.c) * @poll_queue: delayed work for polling * @notify_event: Last notification event * @state: current state of the thermal zone * @debugfs: this thermal zone device's thermal zone debug info * @user_thresholds: list of userspace thresholds for temp. limit notifications * @trips: array of struct thermal_trip objects */ struct thermal_zone_device { int id; char type[THERMAL_NAME_LENGTH]; struct device device; struct completion removal; struct completion resume; struct attribute_group trips_attribute_group; struct list_head trips_high; struct list_head trips_reached; struct list_head trips_invalid; enum thermal_device_mode mode; void *devdata; int num_trips; unsigned long passive_delay_jiffies; unsigned long polling_delay_jiffies; unsigned long recheck_delay_jiffies; int temperature; int last_temperature; int emul_temperature; int passive; int prev_low_trip; int prev_high_trip; struct thermal_zone_device_ops ops; struct thermal_zone_params *tzp; struct thermal_governor *governor; void *governor_data; struct ida ida; struct mutex lock; struct list_head node; struct delayed_work poll_queue; enum thermal_notify_event notify_event; u8 state; #ifdef CONFIG_THERMAL_DEBUGFS struct thermal_debugfs *debugfs; #endif struct list_head user_thresholds; struct thermal_trip_desc trips[] __counted_by(num_trips); }; DEFINE_GUARD(thermal_zone, struct thermal_zone_device *, mutex_lock(&_T->lock), mutex_unlock(&_T->lock)) DEFINE_GUARD(thermal_zone_reverse, struct thermal_zone_device *, mutex_unlock(&_T->lock), mutex_lock(&_T->lock)) /* Initial thermal zone temperature. */ #define THERMAL_TEMP_INIT INT_MIN /* * Default and maximum delay after a failed thermal zone temperature check * before attempting to check it again (in jiffies). */ #define THERMAL_RECHECK_DELAY msecs_to_jiffies(250) #define THERMAL_MAX_RECHECK_DELAY (120 * HZ) /* Default Thermal Governor */ #if defined(CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE) #define DEFAULT_THERMAL_GOVERNOR "step_wise" #elif defined(CONFIG_THERMAL_DEFAULT_GOV_FAIR_SHARE) #define DEFAULT_THERMAL_GOVERNOR "fair_share" #elif defined(CONFIG_THERMAL_DEFAULT_GOV_USER_SPACE) #define DEFAULT_THERMAL_GOVERNOR "user_space" #elif defined(CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR) #define DEFAULT_THERMAL_GOVERNOR "power_allocator" #elif defined(CONFIG_THERMAL_DEFAULT_GOV_BANG_BANG) #define DEFAULT_THERMAL_GOVERNOR "bang_bang" #endif /* Initial state of a cooling device during binding */ #define THERMAL_NO_TARGET -1UL /* Init section thermal table */ extern struct thermal_governor *__governor_thermal_table[]; extern struct thermal_governor *__governor_thermal_table_end[]; #define THERMAL_TABLE_ENTRY(table, name) \ static typeof(name) *__thermal_table_entry_##name \ __used __section("__" #table "_thermal_table") = &name #define THERMAL_GOVERNOR_DECLARE(name) THERMAL_TABLE_ENTRY(governor, name) #define for_each_governor_table(__governor) \ for (__governor = __governor_thermal_table; \ __governor < __governor_thermal_table_end; \ __governor++) int for_each_thermal_zone(int (*cb)(struct thermal_zone_device *, void *), void *); int for_each_thermal_cooling_device(int (*cb)(struct thermal_cooling_device *, void *), void *); int for_each_thermal_governor(int (*cb)(struct thermal_governor *, void *), void *thermal_governor); struct thermal_zone_device *thermal_zone_get_by_id(int id); DEFINE_CLASS(thermal_zone_get_by_id, struct thermal_zone_device *, if (_T) put_device(&_T->device), thermal_zone_get_by_id(id), int id) static inline bool cdev_is_power_actor(struct thermal_cooling_device *cdev) { return cdev->ops->get_requested_power && cdev->ops->state2power && cdev->ops->power2state; } void thermal_cdev_update(struct thermal_cooling_device *); void thermal_cdev_update_nocheck(struct thermal_cooling_device *cdev); void __thermal_cdev_update(struct thermal_cooling_device *cdev); int get_tz_trend(struct thermal_zone_device *tz, const struct thermal_trip *trip); /* * This structure is used to describe the behavior of * a certain cooling device on a certain trip point * in a certain thermal zone */ struct thermal_instance { int id; char name[THERMAL_NAME_LENGTH]; struct thermal_cooling_device *cdev; const struct thermal_trip *trip; bool initialized; unsigned long upper; /* Highest cooling state for this trip point */ unsigned long lower; /* Lowest cooling state for this trip point */ unsigned long target; /* expected cooling state */ char attr_name[THERMAL_NAME_LENGTH]; struct device_attribute attr; char weight_attr_name[THERMAL_NAME_LENGTH]; struct device_attribute weight_attr; struct list_head trip_node; /* node in trip->thermal_instances */ struct list_head cdev_node; /* node in cdev->thermal_instances */ unsigned int weight; /* The weight of the cooling device */ bool upper_no_limit; }; #define to_thermal_zone(_dev) \ container_of(_dev, struct thermal_zone_device, device) #define to_cooling_device(_dev) \ container_of(_dev, struct thermal_cooling_device, device) int thermal_zone_device_set_policy(struct thermal_zone_device *, char *); int thermal_build_list_of_policies(char *buf); void __thermal_zone_device_update(struct thermal_zone_device *tz, enum thermal_notify_event event); void thermal_zone_device_critical_reboot(struct thermal_zone_device *tz); void thermal_zone_device_critical_shutdown(struct thermal_zone_device *tz); void thermal_governor_update_tz(struct thermal_zone_device *tz, enum thermal_notify_event reason); struct thermal_cooling_device * thermal_cooling_device_alloc(const char *type, const struct thermal_cooling_device_ops *ops); int thermal_cooling_device_add(struct thermal_cooling_device *cdev, void *devdata); /* Helpers */ #define for_each_trip_desc(__tz, __td) \ for (__td = __tz->trips; __td - __tz->trips < __tz->num_trips; __td++) #define trip_to_trip_desc(__trip) \ container_of(__trip, struct thermal_trip_desc, trip) const char *thermal_trip_type_name(enum thermal_trip_type trip_type); void thermal_zone_set_trips(struct thermal_zone_device *tz, int low, int high); int thermal_zone_trip_id(const struct thermal_zone_device *tz, const struct thermal_trip *trip); int __thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp); void thermal_zone_set_trip_hyst(struct thermal_zone_device *tz, struct thermal_trip *trip, int hyst); /* sysfs I/F */ int thermal_zone_create_device_groups(struct thermal_zone_device *tz); void thermal_zone_destroy_device_groups(struct thermal_zone_device *); void thermal_cooling_device_setup_sysfs(struct thermal_cooling_device *); void thermal_cooling_device_destroy_sysfs(struct thermal_cooling_device *cdev); void thermal_cooling_device_stats_reinit(struct thermal_cooling_device *cdev); /* used only at binding time */ ssize_t trip_point_show(struct device *, struct device_attribute *, char *); ssize_t weight_show(struct device *, struct device_attribute *, char *); ssize_t weight_store(struct device *, struct device_attribute *, const char *, size_t); #ifdef CONFIG_THERMAL_STATISTICS void thermal_cooling_device_stats_update(struct thermal_cooling_device *cdev, unsigned long new_state); #else static inline void thermal_cooling_device_stats_update(struct thermal_cooling_device *cdev, unsigned long new_state) {} #endif /* CONFIG_THERMAL_STATISTICS */ #endif /* __THERMAL_CORE_H__ */ |
| 13 10 32 25 8 20 20 20 20 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 | #include <linux/gfp.h> #include <linux/initrd.h> #include <linux/ioport.h> #include <linux/swap.h> #include <linux/memblock.h> #include <linux/swapfile.h> #include <linux/swapops.h> #include <linux/kmemleak.h> #include <linux/sched/task.h> #include <linux/execmem.h> #include <asm/set_memory.h> #include <asm/cpu_device_id.h> #include <asm/e820/api.h> #include <asm/init.h> #include <asm/page.h> #include <asm/page_types.h> #include <asm/sections.h> #include <asm/setup.h> #include <asm/tlbflush.h> #include <asm/tlb.h> #include <asm/proto.h> #include <asm/dma.h> /* for MAX_DMA_PFN */ #include <asm/kaslr.h> #include <asm/hypervisor.h> #include <asm/cpufeature.h> #include <asm/pti.h> #include <asm/text-patching.h> #include <asm/memtype.h> #include <asm/mmu_context.h> /* * We need to define the tracepoints somewhere, and tlb.c * is only compiled when SMP=y. */ #define CREATE_TRACE_POINTS #include <trace/events/tlb.h> #include "mm_internal.h" /* * Tables translating between page_cache_type_t and pte encoding. * * The default values are defined statically as minimal supported mode; * WC and WT fall back to UC-. pat_init() updates these values to support * more cache modes, WC and WT, when it is safe to do so. See pat_init() * for the details. Note, __early_ioremap() used during early boot-time * takes pgprot_t (pte encoding) and does not use these tables. * * Index into __cachemode2pte_tbl[] is the cachemode. * * Index into __pte2cachemode_tbl[] are the caching attribute bits of the pte * (_PAGE_PWT, _PAGE_PCD, _PAGE_PAT) at index bit positions 0, 1, 2. */ static uint16_t __cachemode2pte_tbl[_PAGE_CACHE_MODE_NUM] = { [_PAGE_CACHE_MODE_WB ] = 0 | 0 , [_PAGE_CACHE_MODE_WC ] = 0 | _PAGE_PCD, [_PAGE_CACHE_MODE_UC_MINUS] = 0 | _PAGE_PCD, [_PAGE_CACHE_MODE_UC ] = _PAGE_PWT | _PAGE_PCD, [_PAGE_CACHE_MODE_WT ] = 0 | _PAGE_PCD, [_PAGE_CACHE_MODE_WP ] = 0 | _PAGE_PCD, }; unsigned long cachemode2protval(enum page_cache_mode pcm) { if (likely(pcm == 0)) return 0; return __cachemode2pte_tbl[pcm]; } EXPORT_SYMBOL(cachemode2protval); static uint8_t __pte2cachemode_tbl[8] = { [__pte2cm_idx( 0 | 0 | 0 )] = _PAGE_CACHE_MODE_WB, [__pte2cm_idx(_PAGE_PWT | 0 | 0 )] = _PAGE_CACHE_MODE_UC_MINUS, [__pte2cm_idx( 0 | _PAGE_PCD | 0 )] = _PAGE_CACHE_MODE_UC_MINUS, [__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | 0 )] = _PAGE_CACHE_MODE_UC, [__pte2cm_idx( 0 | 0 | _PAGE_PAT)] = _PAGE_CACHE_MODE_WB, [__pte2cm_idx(_PAGE_PWT | 0 | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS, [__pte2cm_idx(0 | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC_MINUS, [__pte2cm_idx(_PAGE_PWT | _PAGE_PCD | _PAGE_PAT)] = _PAGE_CACHE_MODE_UC, }; /* * Check that the write-protect PAT entry is set for write-protect. * To do this without making assumptions how PAT has been set up (Xen has * another layout than the kernel), translate the _PAGE_CACHE_MODE_WP cache * mode via the __cachemode2pte_tbl[] into protection bits (those protection * bits will select a cache mode of WP or better), and then translate the * protection bits back into the cache mode using __pte2cm_idx() and the * __pte2cachemode_tbl[] array. This will return the really used cache mode. */ bool x86_has_pat_wp(void) { uint16_t prot = __cachemode2pte_tbl[_PAGE_CACHE_MODE_WP]; return __pte2cachemode_tbl[__pte2cm_idx(prot)] == _PAGE_CACHE_MODE_WP; } enum page_cache_mode pgprot2cachemode(pgprot_t pgprot) { unsigned long masked; masked = pgprot_val(pgprot) & _PAGE_CACHE_MASK; if (likely(masked == 0)) return 0; return __pte2cachemode_tbl[__pte2cm_idx(masked)]; } static unsigned long __initdata pgt_buf_start; static unsigned long __initdata pgt_buf_end; static unsigned long __initdata pgt_buf_top; static unsigned long min_pfn_mapped; static bool __initdata can_use_brk_pgt = true; /* * Pages returned are already directly mapped. * * Changing that is likely to break Xen, see commit: * * 279b706 x86,xen: introduce x86_init.mapping.pagetable_reserve * * for detailed information. */ __ref void *alloc_low_pages(unsigned int num) { unsigned long pfn; int i; if (after_bootmem) { unsigned int order; order = get_order((unsigned long)num << PAGE_SHIFT); return (void *)__get_free_pages(GFP_ATOMIC | __GFP_ZERO, order); } if ((pgt_buf_end + num) > pgt_buf_top || !can_use_brk_pgt) { unsigned long ret = 0; if (min_pfn_mapped < max_pfn_mapped) { ret = memblock_phys_alloc_range( PAGE_SIZE * num, PAGE_SIZE, min_pfn_mapped << PAGE_SHIFT, max_pfn_mapped << PAGE_SHIFT); } if (!ret && can_use_brk_pgt) ret = __pa(extend_brk(PAGE_SIZE * num, PAGE_SIZE)); if (!ret) panic("alloc_low_pages: can not alloc memory"); pfn = ret >> PAGE_SHIFT; } else { pfn = pgt_buf_end; pgt_buf_end += num; } for (i = 0; i < num; i++) { void *adr; adr = __va((pfn + i) << PAGE_SHIFT); clear_page(adr); } return __va(pfn << PAGE_SHIFT); } /* * By default need to be able to allocate page tables below PGD firstly for * the 0-ISA_END_ADDRESS range and secondly for the initial PMD_SIZE mapping. * With KASLR memory randomization, depending on the machine e820 memory and the * PUD alignment, twice that many pages may be needed when KASLR memory * randomization is enabled. */ #define INIT_PGD_PAGE_TABLES 4 #ifndef CONFIG_RANDOMIZE_MEMORY #define INIT_PGD_PAGE_COUNT (2 * INIT_PGD_PAGE_TABLES) #else #define INIT_PGD_PAGE_COUNT (4 * INIT_PGD_PAGE_TABLES) #endif #define INIT_PGT_BUF_SIZE (INIT_PGD_PAGE_COUNT * PAGE_SIZE) RESERVE_BRK(early_pgt_alloc, INIT_PGT_BUF_SIZE); void __init early_alloc_pgt_buf(void) { unsigned long tables = INIT_PGT_BUF_SIZE; phys_addr_t base; base = __pa(extend_brk(tables, PAGE_SIZE)); pgt_buf_start = base >> PAGE_SHIFT; pgt_buf_end = pgt_buf_start; pgt_buf_top = pgt_buf_start + (tables >> PAGE_SHIFT); } int after_bootmem; early_param_on_off("gbpages", "nogbpages", direct_gbpages, CONFIG_X86_DIRECT_GBPAGES); struct map_range { unsigned long start; unsigned long end; unsigned page_size_mask; }; static int page_size_mask; /* * Save some of cr4 feature set we're using (e.g. Pentium 4MB * enable and PPro Global page enable), so that any CPU's that boot * up after us can get the correct flags. Invoked on the boot CPU. */ static inline void cr4_set_bits_and_update_boot(unsigned long mask) { mmu_cr4_features |= mask; if (trampoline_cr4_features) *trampoline_cr4_features = mmu_cr4_features; cr4_set_bits(mask); } static void __init probe_page_size_mask(void) { /* * For pagealloc debugging, identity mapping will use small pages. * This will simplify cpa(), which otherwise needs to support splitting * large pages into small in interrupt context, etc. */ if (boot_cpu_has(X86_FEATURE_PSE) && !debug_pagealloc_enabled()) page_size_mask |= 1 << PG_LEVEL_2M; else direct_gbpages = 0; /* Enable PSE if available */ if (boot_cpu_has(X86_FEATURE_PSE)) cr4_set_bits_and_update_boot(X86_CR4_PSE); /* Enable PGE if available */ __supported_pte_mask &= ~_PAGE_GLOBAL; if (boot_cpu_has(X86_FEATURE_PGE)) { cr4_set_bits_and_update_boot(X86_CR4_PGE); __supported_pte_mask |= _PAGE_GLOBAL; } /* By the default is everything supported: */ __default_kernel_pte_mask = __supported_pte_mask; /* Except when with PTI where the kernel is mostly non-Global: */ if (cpu_feature_enabled(X86_FEATURE_PTI)) __default_kernel_pte_mask &= ~_PAGE_GLOBAL; /* Enable 1 GB linear kernel mappings if available: */ if (direct_gbpages && boot_cpu_has(X86_FEATURE_GBPAGES)) { printk(KERN_INFO "Using GB pages for direct mapping\n"); page_size_mask |= 1 << PG_LEVEL_1G; } else { direct_gbpages = 0; } } /* * INVLPG may not properly flush Global entries on * these CPUs. New microcode fixes the issue. */ static const struct x86_cpu_id invlpg_miss_ids[] = { X86_MATCH_VFM(INTEL_ALDERLAKE, 0x2e), X86_MATCH_VFM(INTEL_ALDERLAKE_L, 0x42c), X86_MATCH_VFM(INTEL_ATOM_GRACEMONT, 0x11), X86_MATCH_VFM(INTEL_RAPTORLAKE, 0x118), X86_MATCH_VFM(INTEL_RAPTORLAKE_P, 0x4117), X86_MATCH_VFM(INTEL_RAPTORLAKE_S, 0x2e), {} }; static void setup_pcid(void) { const struct x86_cpu_id *invlpg_miss_match; if (!IS_ENABLED(CONFIG_X86_64)) return; if (!boot_cpu_has(X86_FEATURE_PCID)) return; invlpg_miss_match = x86_match_cpu(invlpg_miss_ids); if (invlpg_miss_match && boot_cpu_data.microcode < invlpg_miss_match->driver_data) { pr_info("Incomplete global flushes, disabling PCID"); setup_clear_cpu_cap(X86_FEATURE_PCID); return; } if (boot_cpu_has(X86_FEATURE_PGE)) { /* * This can't be cr4_set_bits_and_update_boot() -- the * trampoline code can't handle CR4.PCIDE and it wouldn't * do any good anyway. Despite the name, * cr4_set_bits_and_update_boot() doesn't actually cause * the bits in question to remain set all the way through * the secondary boot asm. * * Instead, we brute-force it and set CR4.PCIDE manually in * start_secondary(). */ cr4_set_bits(X86_CR4_PCIDE); } else { /* * flush_tlb_all(), as currently implemented, won't work if * PCID is on but PGE is not. Since that combination * doesn't exist on real hardware, there's no reason to try * to fully support it, but it's polite to avoid corrupting * data if we're on an improperly configured VM. */ setup_clear_cpu_cap(X86_FEATURE_PCID); } } #ifdef CONFIG_X86_32 #define NR_RANGE_MR 3 #else /* CONFIG_X86_64 */ #define NR_RANGE_MR 5 #endif static int __meminit save_mr(struct map_range *mr, int nr_range, unsigned long start_pfn, unsigned long end_pfn, unsigned long page_size_mask) { if (start_pfn < end_pfn) { if (nr_range >= NR_RANGE_MR) panic("run out of range for init_memory_mapping\n"); mr[nr_range].start = start_pfn<<PAGE_SHIFT; mr[nr_range].end = end_pfn<<PAGE_SHIFT; mr[nr_range].page_size_mask = page_size_mask; nr_range++; } return nr_range; } /* * adjust the page_size_mask for small range to go with * big page size instead small one if nearby are ram too. */ static void __ref adjust_range_page_size_mask(struct map_range *mr, int nr_range) { int i; for (i = 0; i < nr_range; i++) { if ((page_size_mask & (1<<PG_LEVEL_2M)) && !(mr[i].page_size_mask & (1<<PG_LEVEL_2M))) { unsigned long start = round_down(mr[i].start, PMD_SIZE); unsigned long end = round_up(mr[i].end, PMD_SIZE); #ifdef CONFIG_X86_32 if ((end >> PAGE_SHIFT) > max_low_pfn) continue; #endif if (memblock_is_region_memory(start, end - start)) mr[i].page_size_mask |= 1<<PG_LEVEL_2M; } if ((page_size_mask & (1<<PG_LEVEL_1G)) && !(mr[i].page_size_mask & (1<<PG_LEVEL_1G))) { unsigned long start = round_down(mr[i].start, PUD_SIZE); unsigned long end = round_up(mr[i].end, PUD_SIZE); if (memblock_is_region_memory(start, end - start)) mr[i].page_size_mask |= 1<<PG_LEVEL_1G; } } } static const char *page_size_string(struct map_range *mr) { static const char str_1g[] = "1G"; static const char str_2m[] = "2M"; static const char str_4m[] = "4M"; static const char str_4k[] = "4k"; if (mr->page_size_mask & (1<<PG_LEVEL_1G)) return str_1g; /* * 32-bit without PAE has a 4M large page size. * PG_LEVEL_2M is misnamed, but we can at least * print out the right size in the string. */ if (IS_ENABLED(CONFIG_X86_32) && !IS_ENABLED(CONFIG_X86_PAE) && mr->page_size_mask & (1<<PG_LEVEL_2M)) return str_4m; if (mr->page_size_mask & (1<<PG_LEVEL_2M)) return str_2m; return str_4k; } static int __meminit split_mem_range(struct map_range *mr, int nr_range, unsigned long start, unsigned long end) { unsigned long start_pfn, end_pfn, limit_pfn; unsigned long pfn; int i; limit_pfn = PFN_DOWN(end); /* head if not big page alignment ? */ pfn = start_pfn = PFN_DOWN(start); #ifdef CONFIG_X86_32 /* * Don't use a large page for the first 2/4MB of memory * because there are often fixed size MTRRs in there * and overlapping MTRRs into large pages can cause * slowdowns. */ if (pfn == 0) end_pfn = PFN_DOWN(PMD_SIZE); else end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); #else /* CONFIG_X86_64 */ end_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); #endif if (end_pfn > limit_pfn) end_pfn = limit_pfn; if (start_pfn < end_pfn) { nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); pfn = end_pfn; } /* big page (2M) range */ start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); #ifdef CONFIG_X86_32 end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); #else /* CONFIG_X86_64 */ end_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE)); if (end_pfn > round_down(limit_pfn, PFN_DOWN(PMD_SIZE))) end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); #endif if (start_pfn < end_pfn) { nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, page_size_mask & (1<<PG_LEVEL_2M)); pfn = end_pfn; } #ifdef CONFIG_X86_64 /* big page (1G) range */ start_pfn = round_up(pfn, PFN_DOWN(PUD_SIZE)); end_pfn = round_down(limit_pfn, PFN_DOWN(PUD_SIZE)); if (start_pfn < end_pfn) { nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, page_size_mask & ((1<<PG_LEVEL_2M)|(1<<PG_LEVEL_1G))); pfn = end_pfn; } /* tail is not big page (1G) alignment */ start_pfn = round_up(pfn, PFN_DOWN(PMD_SIZE)); end_pfn = round_down(limit_pfn, PFN_DOWN(PMD_SIZE)); if (start_pfn < end_pfn) { nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, page_size_mask & (1<<PG_LEVEL_2M)); pfn = end_pfn; } #endif /* tail is not big page (2M) alignment */ start_pfn = pfn; end_pfn = limit_pfn; nr_range = save_mr(mr, nr_range, start_pfn, end_pfn, 0); if (!after_bootmem) adjust_range_page_size_mask(mr, nr_range); /* try to merge same page size and continuous */ for (i = 0; nr_range > 1 && i < nr_range - 1; i++) { unsigned long old_start; if (mr[i].end != mr[i+1].start || mr[i].page_size_mask != mr[i+1].page_size_mask) continue; /* move it */ old_start = mr[i].start; memmove(&mr[i], &mr[i+1], (nr_range - 1 - i) * sizeof(struct map_range)); mr[i--].start = old_start; nr_range--; } for (i = 0; i < nr_range; i++) pr_debug(" [mem %#010lx-%#010lx] page %s\n", mr[i].start, mr[i].end - 1, page_size_string(&mr[i])); return nr_range; } struct range pfn_mapped[E820_MAX_ENTRIES]; int nr_pfn_mapped; static void add_pfn_range_mapped(unsigned long start_pfn, unsigned long end_pfn) { nr_pfn_mapped = add_range_with_merge(pfn_mapped, E820_MAX_ENTRIES, nr_pfn_mapped, start_pfn, end_pfn); nr_pfn_mapped = clean_sort_range(pfn_mapped, E820_MAX_ENTRIES); max_pfn_mapped = max(max_pfn_mapped, end_pfn); if (start_pfn < (1UL<<(32-PAGE_SHIFT))) max_low_pfn_mapped = max(max_low_pfn_mapped, min(end_pfn, 1UL<<(32-PAGE_SHIFT))); } bool pfn_range_is_mapped(unsigned long start_pfn, unsigned long end_pfn) { int i; for (i = 0; i < nr_pfn_mapped; i++) if ((start_pfn >= pfn_mapped[i].start) && (end_pfn <= pfn_mapped[i].end)) return true; return false; } /* * Setup the direct mapping of the physical memory at PAGE_OFFSET. * This runs before bootmem is initialized and gets pages directly from * the physical memory. To access them they are temporarily mapped. */ unsigned long __ref init_memory_mapping(unsigned long start, unsigned long end, pgprot_t prot) { struct map_range mr[NR_RANGE_MR]; unsigned long ret = 0; int nr_range, i; pr_debug("init_memory_mapping: [mem %#010lx-%#010lx]\n", start, end - 1); memset(mr, 0, sizeof(mr)); nr_range = split_mem_range(mr, 0, start, end); for (i = 0; i < nr_range; i++) ret = kernel_physical_mapping_init(mr[i].start, mr[i].end, mr[i].page_size_mask, prot); add_pfn_range_mapped(start >> PAGE_SHIFT, ret >> PAGE_SHIFT); return ret >> PAGE_SHIFT; } /* * We need to iterate through the E820 memory map and create direct mappings * for only E820_TYPE_RAM and E820_KERN_RESERVED regions. We cannot simply * create direct mappings for all pfns from [0 to max_low_pfn) and * [4GB to max_pfn) because of possible memory holes in high addresses * that cannot be marked as UC by fixed/variable range MTRRs. * Depending on the alignment of E820 ranges, this may possibly result * in using smaller size (i.e. 4K instead of 2M or 1G) page tables. * * init_mem_mapping() calls init_range_memory_mapping() with big range. * That range would have hole in the middle or ends, and only ram parts * will be mapped in init_range_memory_mapping(). */ static unsigned long __init init_range_memory_mapping( unsigned long r_start, unsigned long r_end) { unsigned long start_pfn, end_pfn; unsigned long mapped_ram_size = 0; int i; for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, NULL) { u64 start = clamp_val(PFN_PHYS(start_pfn), r_start, r_end); u64 end = clamp_val(PFN_PHYS(end_pfn), r_start, r_end); if (start >= end) continue; /* * if it is overlapping with brk pgt, we need to * alloc pgt buf from memblock instead. */ can_use_brk_pgt = max(start, (u64)pgt_buf_end<<PAGE_SHIFT) >= min(end, (u64)pgt_buf_top<<PAGE_SHIFT); init_memory_mapping(start, end, PAGE_KERNEL); mapped_ram_size += end - start; can_use_brk_pgt = true; } return mapped_ram_size; } static unsigned long __init get_new_step_size(unsigned long step_size) { /* * Initial mapped size is PMD_SIZE (2M). * We can not set step_size to be PUD_SIZE (1G) yet. * In worse case, when we cross the 1G boundary, and * PG_LEVEL_2M is not set, we will need 1+1+512 pages (2M + 8k) * to map 1G range with PTE. Hence we use one less than the * difference of page table level shifts. * * Don't need to worry about overflow in the top-down case, on 32bit, * when step_size is 0, round_down() returns 0 for start, and that * turns it into 0x100000000ULL. * In the bottom-up case, round_up(x, 0) returns 0 though too, which * needs to be taken into consideration by the code below. */ return step_size << (PMD_SHIFT - PAGE_SHIFT - 1); } /** * memory_map_top_down - Map [map_start, map_end) top down * @map_start: start address of the target memory range * @map_end: end address of the target memory range * * This function will setup direct mapping for memory range * [map_start, map_end) in top-down. That said, the page tables * will be allocated at the end of the memory, and we map the * memory in top-down. */ static void __init memory_map_top_down(unsigned long map_start, unsigned long map_end) { unsigned long real_end, last_start; unsigned long step_size; unsigned long addr; unsigned long mapped_ram_size = 0; /* * Systems that have many reserved areas near top of the memory, * e.g. QEMU with less than 1G RAM and EFI enabled, or Xen, will * require lots of 4K mappings which may exhaust pgt_buf. * Start with top-most PMD_SIZE range aligned at PMD_SIZE to ensure * there is enough mapped memory that can be allocated from * memblock. */ addr = memblock_phys_alloc_range(PMD_SIZE, PMD_SIZE, map_start, map_end); if (!addr) { pr_warn("Failed to release memory for alloc_low_pages()"); real_end = max(map_start, ALIGN_DOWN(map_end, PMD_SIZE)); } else { memblock_phys_free(addr, PMD_SIZE); real_end = addr + PMD_SIZE; } /* step_size need to be small so pgt_buf from BRK could cover it */ step_size = PMD_SIZE; max_pfn_mapped = 0; /* will get exact value next */ min_pfn_mapped = real_end >> PAGE_SHIFT; last_start = real_end; /* * We start from the top (end of memory) and go to the bottom. * The memblock_find_in_range() gets us a block of RAM from the * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages * for page table. */ while (last_start > map_start) { unsigned long start; if (last_start > step_size) { start = round_down(last_start - 1, step_size); if (start < map_start) start = map_start; } else start = map_start; mapped_ram_size += init_range_memory_mapping(start, last_start); last_start = start; min_pfn_mapped = last_start >> PAGE_SHIFT; if (mapped_ram_size >= step_size) step_size = get_new_step_size(step_size); } if (real_end < map_end) init_range_memory_mapping(real_end, map_end); } /** * memory_map_bottom_up - Map [map_start, map_end) bottom up * @map_start: start address of the target memory range * @map_end: end address of the target memory range * * This function will setup direct mapping for memory range * [map_start, map_end) in bottom-up. Since we have limited the * bottom-up allocation above the kernel, the page tables will * be allocated just above the kernel and we map the memory * in [map_start, map_end) in bottom-up. */ static void __init memory_map_bottom_up(unsigned long map_start, unsigned long map_end) { unsigned long next, start; unsigned long mapped_ram_size = 0; /* step_size need to be small so pgt_buf from BRK could cover it */ unsigned long step_size = PMD_SIZE; start = map_start; min_pfn_mapped = start >> PAGE_SHIFT; /* * We start from the bottom (@map_start) and go to the top (@map_end). * The memblock_find_in_range() gets us a block of RAM from the * end of RAM in [min_pfn_mapped, max_pfn_mapped) used as new pages * for page table. */ while (start < map_end) { if (step_size && map_end - start > step_size) { next = round_up(start + 1, step_size); if (next > map_end) next = map_end; } else { next = map_end; } mapped_ram_size += init_range_memory_mapping(start, next); start = next; if (mapped_ram_size >= step_size) step_size = get_new_step_size(step_size); } } /* * The real mode trampoline, which is required for bootstrapping CPUs * occupies only a small area under the low 1MB. See reserve_real_mode() * for details. * * If KASLR is disabled the first PGD entry of the direct mapping is copied * to map the real mode trampoline. * * If KASLR is enabled, copy only the PUD which covers the low 1MB * area. This limits the randomization granularity to 1GB for both 4-level * and 5-level paging. */ static void __init init_trampoline(void) { #ifdef CONFIG_X86_64 /* * The code below will alias kernel page-tables in the user-range of the * address space, including the Global bit. So global TLB entries will * be created when using the trampoline page-table. */ if (!kaslr_memory_enabled()) trampoline_pgd_entry = init_top_pgt[pgd_index(__PAGE_OFFSET)]; else init_trampoline_kaslr(); #endif } void __init init_mem_mapping(void) { unsigned long end; pti_check_boottime_disable(); probe_page_size_mask(); setup_pcid(); #ifdef CONFIG_X86_64 end = max_pfn << PAGE_SHIFT; #else end = max_low_pfn << PAGE_SHIFT; #endif /* the ISA range is always mapped regardless of memory holes */ init_memory_mapping(0, ISA_END_ADDRESS, PAGE_KERNEL); /* Init the trampoline, possibly with KASLR memory offset */ init_trampoline(); /* * If the allocation is in bottom-up direction, we setup direct mapping * in bottom-up, otherwise we setup direct mapping in top-down. */ if (memblock_bottom_up()) { unsigned long kernel_end = __pa_symbol(_end); /* * we need two separate calls here. This is because we want to * allocate page tables above the kernel. So we first map * [kernel_end, end) to make memory above the kernel be mapped * as soon as possible. And then use page tables allocated above * the kernel to map [ISA_END_ADDRESS, kernel_end). */ memory_map_bottom_up(kernel_end, end); memory_map_bottom_up(ISA_END_ADDRESS, kernel_end); } else { memory_map_top_down(ISA_END_ADDRESS, end); } #ifdef CONFIG_X86_64 if (max_pfn > max_low_pfn) { /* can we preserve max_low_pfn ?*/ max_low_pfn = max_pfn; } #else early_ioremap_page_table_range_init(); #endif load_cr3(swapper_pg_dir); __flush_tlb_all(); x86_init.hyper.init_mem_mapping(); early_memtest(0, max_pfn_mapped << PAGE_SHIFT); } /* * Initialize an mm_struct to be used during poking and a pointer to be used * during patching. */ void __init poking_init(void) { spinlock_t *ptl; pte_t *ptep; text_poke_mm = mm_alloc(); BUG_ON(!text_poke_mm); /* Xen PV guests need the PGD to be pinned. */ paravirt_enter_mmap(text_poke_mm); set_notrack_mm(text_poke_mm); /* * Randomize the poking address, but make sure that the following page * will be mapped at the same PMD. We need 2 pages, so find space for 3, * and adjust the address if the PMD ends after the first one. */ text_poke_mm_addr = TASK_UNMAPPED_BASE; if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) text_poke_mm_addr += (kaslr_get_random_long("Poking") & PAGE_MASK) % (TASK_SIZE - TASK_UNMAPPED_BASE - 3 * PAGE_SIZE); if (((text_poke_mm_addr + PAGE_SIZE) & ~PMD_MASK) == 0) text_poke_mm_addr += PAGE_SIZE; /* * We need to trigger the allocation of the page-tables that will be * needed for poking now. Later, poking may be performed in an atomic * section, which might cause allocation to fail. */ ptep = get_locked_pte(text_poke_mm, text_poke_mm_addr, &ptl); BUG_ON(!ptep); pte_unmap_unlock(ptep, ptl); } /* * devmem_is_allowed() checks to see if /dev/mem access to a certain address * is valid. The argument is a physical page number. * * On x86, access has to be given to the first megabyte of RAM because that * area traditionally contains BIOS code and data regions used by X, dosemu, * and similar apps. Since they map the entire memory range, the whole range * must be allowed (for mapping), but any areas that would otherwise be * disallowed are flagged as being "zero filled" instead of rejected. * Access has to be given to non-kernel-ram areas as well, these contain the * PCI mmio resources as well as potential bios/acpi data regions. */ int devmem_is_allowed(unsigned long pagenr) { if (region_intersects(PFN_PHYS(pagenr), PAGE_SIZE, IORESOURCE_SYSTEM_RAM, IORES_DESC_NONE) != REGION_DISJOINT) { /* * For disallowed memory regions in the low 1MB range, * request that the page be shown as all zeros. */ if (pagenr < 256) return 2; return 0; } /* * This must follow RAM test, since System RAM is considered a * restricted resource under CONFIG_STRICT_DEVMEM. */ if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) { /* Low 1MB bypasses iomem restrictions. */ if (pagenr < 256) return 1; return 0; } return 1; } void free_init_pages(const char *what, unsigned long begin, unsigned long end) { unsigned long begin_aligned, end_aligned; /* Make sure boundaries are page aligned */ begin_aligned = PAGE_ALIGN(begin); end_aligned = end & PAGE_MASK; if (WARN_ON(begin_aligned != begin || end_aligned != end)) { begin = begin_aligned; end = end_aligned; } if (begin >= end) return; /* * If debugging page accesses then do not free this memory but * mark them not present - any buggy init-section access will * create a kernel page fault: */ if (debug_pagealloc_enabled()) { pr_info("debug: unmapping init [mem %#010lx-%#010lx]\n", begin, end - 1); /* * Inform kmemleak about the hole in the memory since the * corresponding pages will be unmapped. */ kmemleak_free_part((void *)begin, end - begin); set_memory_np(begin, (end - begin) >> PAGE_SHIFT); } else { /* * We just marked the kernel text read only above, now that * we are going to free part of that, we need to make that * writeable and non-executable first. */ set_memory_nx(begin, (end - begin) >> PAGE_SHIFT); set_memory_rw(begin, (end - begin) >> PAGE_SHIFT); free_reserved_area((void *)begin, (void *)end, POISON_FREE_INITMEM, what); } } /* * begin/end can be in the direct map or the "high kernel mapping" * used for the kernel image only. free_init_pages() will do the * right thing for either kind of address. */ void free_kernel_image_pages(const char *what, void *begin, void *end) { unsigned long begin_ul = (unsigned long)begin; unsigned long end_ul = (unsigned long)end; unsigned long len_pages = (end_ul - begin_ul) >> PAGE_SHIFT; free_init_pages(what, begin_ul, end_ul); /* * PTI maps some of the kernel into userspace. For performance, * this includes some kernel areas that do not contain secrets. * Those areas might be adjacent to the parts of the kernel image * being freed, which may contain secrets. Remove the "high kernel * image mapping" for these freed areas, ensuring they are not even * potentially vulnerable to Meltdown regardless of the specific * optimizations PTI is currently using. * * The "noalias" prevents unmapping the direct map alias which is * needed to access the freed pages. * * This is only valid for 64bit kernels. 32bit has only one mapping * which can't be treated in this way for obvious reasons. */ if (IS_ENABLED(CONFIG_X86_64) && cpu_feature_enabled(X86_FEATURE_PTI)) set_memory_np_noalias(begin_ul, len_pages); } void __ref free_initmem(void) { e820__reallocate_tables(); mem_encrypt_free_decrypted_mem(); free_kernel_image_pages("unused kernel image (initmem)", &__init_begin, &__init_end); } #ifdef CONFIG_BLK_DEV_INITRD void __init free_initrd_mem(unsigned long start, unsigned long end) { /* * end could be not aligned, and We can not align that, * decompressor could be confused by aligned initrd_end * We already reserve the end partial page before in * - i386_start_kernel() * - x86_64_start_kernel() * - relocate_initrd() * So here We can do PAGE_ALIGN() safely to get partial page to be freed */ free_init_pages("initrd", start, PAGE_ALIGN(end)); } #endif void __init arch_zone_limits_init(unsigned long *max_zone_pfns) { #ifdef CONFIG_ZONE_DMA max_zone_pfns[ZONE_DMA] = min(MAX_DMA_PFN, max_low_pfn); #endif #ifdef CONFIG_ZONE_DMA32 max_zone_pfns[ZONE_DMA32] = min(MAX_DMA32_PFN, max_low_pfn); #endif max_zone_pfns[ZONE_NORMAL] = max_low_pfn; #ifdef CONFIG_HIGHMEM max_zone_pfns[ZONE_HIGHMEM] = max_pfn; #endif } __visible DEFINE_PER_CPU_ALIGNED(struct tlb_state, cpu_tlbstate) = { .loaded_mm = &init_mm, .next_asid = 1, .cr4 = ~0UL, /* fail hard if we screw up cr4 shadow initialization */ }; #ifdef CONFIG_ADDRESS_MASKING DEFINE_PER_CPU(u64, tlbstate_untag_mask); EXPORT_PER_CPU_SYMBOL(tlbstate_untag_mask); #endif void update_cache_mode_entry(unsigned entry, enum page_cache_mode cache) { /* entry 0 MUST be WB (hardwired to speed up translations) */ BUG_ON(!entry && cache != _PAGE_CACHE_MODE_WB); __cachemode2pte_tbl[cache] = __cm_idx2pte(entry); __pte2cachemode_tbl[entry] = cache; } #ifdef CONFIG_SWAP unsigned long arch_max_swapfile_size(void) { unsigned long pages; pages = generic_max_swapfile_size(); if (boot_cpu_has_bug(X86_BUG_L1TF) && l1tf_mitigation != L1TF_MITIGATION_OFF) { /* Limit the swap file size to MAX_PA/2 for L1TF workaround */ unsigned long long l1tf_limit = l1tf_pfn_limit(); /* * We encode swap offsets also with 3 bits below those for pfn * which makes the usable limit higher. */ #if CONFIG_PGTABLE_LEVELS > 2 l1tf_limit <<= PAGE_SHIFT - SWP_OFFSET_FIRST_BIT; #endif pages = min_t(unsigned long long, l1tf_limit, pages); } return pages; } #endif #ifdef CONFIG_EXECMEM static struct execmem_info execmem_info __ro_after_init; #ifdef CONFIG_ARCH_HAS_EXECMEM_ROX void execmem_fill_trapping_insns(void *ptr, size_t size) { memset(ptr, INT3_INSN_OPCODE, size); } #endif struct execmem_info __init *execmem_arch_setup(void) { unsigned long start, offset = 0; enum execmem_range_flags flags; pgprot_t pgprot; if (kaslr_enabled()) offset = get_random_u32_inclusive(1, 1024) * PAGE_SIZE; start = MODULES_VADDR + offset; if (IS_ENABLED(CONFIG_ARCH_HAS_EXECMEM_ROX) && cpu_feature_enabled(X86_FEATURE_PSE)) { pgprot = PAGE_KERNEL_ROX; flags = EXECMEM_KASAN_SHADOW | EXECMEM_ROX_CACHE; } else { pgprot = PAGE_KERNEL; flags = EXECMEM_KASAN_SHADOW; } execmem_info = (struct execmem_info){ .ranges = { [EXECMEM_MODULE_TEXT] = { .flags = flags, .start = start, .end = MODULES_END, .pgprot = pgprot, .alignment = MODULE_ALIGN, }, [EXECMEM_KPROBES] = { .flags = flags, .start = start, .end = MODULES_END, .pgprot = PAGE_KERNEL_ROX, .alignment = MODULE_ALIGN, }, [EXECMEM_FTRACE] = { .flags = flags, .start = start, .end = MODULES_END, .pgprot = pgprot, .alignment = MODULE_ALIGN, }, [EXECMEM_BPF] = { .flags = EXECMEM_KASAN_SHADOW, .start = start, .end = MODULES_END, .pgprot = PAGE_KERNEL, .alignment = MODULE_ALIGN, }, [EXECMEM_MODULE_DATA] = { .flags = EXECMEM_KASAN_SHADOW, .start = start, .end = MODULES_END, .pgprot = PAGE_KERNEL, .alignment = MODULE_ALIGN, }, }, }; return &execmem_info; } #endif /* CONFIG_EXECMEM */ |
| 14 9 7 5 5 5 3 16 18 17 16 16 18 16 7 3 3 3 1 1 1 1 2 3 3 3 6 18 18 17 18 19 19 13 5 17 3 3 1 15 17 17 17 17 17 18 11 10 9 3 6 7 1 1 1 1 6 5 2 1 5 3 1 1 1 1 4 4 5 1 11 1 1 13 12 11 10 10 9 9 9 9 8 3 3 1 9 1 1 4 4 3 2 2 2 2 4 8 8 7 4 5 5 4 3 3 2 2 2 2 2 2 4 3 2 2 4 6 3 5 2 3 5 5 2 2 5 5 5 5 5 5 5 1 4 3 2 3 5 2 17 12 12 14 10 14 9 14 14 14 14 14 14 14 14 14 12 4 11 5 7 3 5 5 18 18 2 1 1 2 1 19 18 18 1 1 2 1 2 1 1 1 2 1 2 1 2 1 2 1 2 1 1 19 12 11 10 8 1 1 1 1 1 1 1 12 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 | // SPDX-License-Identifier: GPL-2.0 /* * af_llc.c - LLC User Interface SAPs * Description: * Functions in this module are implementation of socket based llc * communications for the Linux operating system. Support of llc class * one and class two is provided via SOCK_DGRAM and SOCK_STREAM * respectively. * * An llc2 connection is (mac + sap), only one llc2 sap connection * is allowed per mac. Though one sap may have multiple mac + sap * connections. * * Copyright (c) 2001 by Jay Schulist <jschlst@samba.org> * 2002-2003 by Arnaldo Carvalho de Melo <acme@conectiva.com.br> */ #include <linux/compiler.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/rtnetlink.h> #include <linux/init.h> #include <linux/slab.h> #include <linux/sched/signal.h> #include <linux/uio.h> #include <net/llc.h> #include <net/llc_sap.h> #include <net/llc_pdu.h> #include <net/llc_conn.h> #include <net/tcp_states.h> /* remember: uninitialized global data is zeroed because its in .bss */ static u16 llc_ui_sap_last_autoport = LLC_SAP_DYN_START; static u16 llc_ui_sap_link_no_max[256]; static struct sockaddr_llc llc_ui_addrnull; static const struct proto_ops llc_ui_ops; static bool llc_ui_wait_for_conn(struct sock *sk, long timeout); static int llc_ui_wait_for_disc(struct sock *sk, long timeout); static int llc_ui_wait_for_busy_core(struct sock *sk, long timeout); #if 0 #define dprintk(args...) printk(KERN_DEBUG args) #else #define dprintk(args...) do {} while (0) #endif /* Maybe we'll add some more in the future. */ #define LLC_CMSG_PKTINFO 1 /** * llc_ui_next_link_no - return the next unused link number for a sap * @sap: Address of sap to get link number from. * * Return the next unused link number for a given sap. */ static inline u16 llc_ui_next_link_no(int sap) { return llc_ui_sap_link_no_max[sap]++; } /** * llc_proto_type - return eth protocol for ARP header type * @arphrd: ARP header type. * * Given an ARP header type return the corresponding ethernet protocol. */ static inline __be16 llc_proto_type(u16 arphrd) { return htons(ETH_P_802_2); } /** * llc_ui_addr_null - determines if a address structure is null * @addr: Address to test if null. */ static inline u8 llc_ui_addr_null(struct sockaddr_llc *addr) { return !memcmp(addr, &llc_ui_addrnull, sizeof(*addr)); } /** * llc_ui_header_len - return length of llc header based on operation * @sk: Socket which contains a valid llc socket type. * @addr: Complete sockaddr_llc structure received from the user. * * Provide the length of the llc header depending on what kind of * operation the user would like to perform and the type of socket. * Returns the correct llc header length. */ static inline u8 llc_ui_header_len(struct sock *sk, struct sockaddr_llc *addr) { u8 rc = LLC_PDU_LEN_U; if (addr->sllc_test) rc = LLC_PDU_LEN_U; else if (addr->sllc_xid) /* We need to expand header to sizeof(struct llc_xid_info) * since llc_pdu_init_as_xid_cmd() sets 4,5,6 bytes of LLC header * as XID PDU. In llc_ui_sendmsg() we reserved header size and then * filled all other space with user data. If we won't reserve this * bytes, llc_pdu_init_as_xid_cmd() will overwrite user data */ rc = LLC_PDU_LEN_U_XID; else if (sk->sk_type == SOCK_STREAM) rc = LLC_PDU_LEN_I; return rc; } /** * llc_ui_send_data - send data via reliable llc2 connection * @sk: Connection the socket is using. * @skb: Data the user wishes to send. * @noblock: can we block waiting for data? * * Send data via reliable llc2 connection. * Returns 0 upon success, non-zero if action did not succeed. * * This function always consumes a reference to the skb. */ static int llc_ui_send_data(struct sock* sk, struct sk_buff *skb, int noblock) { struct llc_sock* llc = llc_sk(sk); if (unlikely(llc_data_accept_state(llc->state) || llc->remote_busy_flag || llc->p_flag)) { long timeout = sock_sndtimeo(sk, noblock); int rc; rc = llc_ui_wait_for_busy_core(sk, timeout); if (rc) { kfree_skb(skb); return rc; } } return llc_build_and_send_pkt(sk, skb); } static void llc_ui_sk_init(struct socket *sock, struct sock *sk) { sock_graft(sk, sock); sk->sk_type = sock->type; sock->ops = &llc_ui_ops; } static struct proto llc_proto = { .name = "LLC", .owner = THIS_MODULE, .obj_size = sizeof(struct llc_sock), .slab_flags = SLAB_TYPESAFE_BY_RCU, }; /** * llc_ui_create - alloc and init a new llc_ui socket * @net: network namespace (must be default network) * @sock: Socket to initialize and attach allocated sk to. * @protocol: Unused. * @kern: on behalf of kernel or userspace * * Allocate and initialize a new llc_ui socket, validate the user wants a * socket type we have available. * Returns 0 upon success, negative upon failure. */ static int llc_ui_create(struct net *net, struct socket *sock, int protocol, int kern) { struct sock *sk; int rc = -ESOCKTNOSUPPORT; if (!ns_capable(net->user_ns, CAP_NET_RAW)) return -EPERM; if (!net_eq(net, &init_net)) return -EAFNOSUPPORT; if (likely(sock->type == SOCK_DGRAM || sock->type == SOCK_STREAM)) { rc = -ENOMEM; sk = llc_sk_alloc(net, PF_LLC, GFP_KERNEL, &llc_proto, kern); if (sk) { rc = 0; llc_ui_sk_init(sock, sk); } } return rc; } /** * llc_ui_release - shutdown socket * @sock: Socket to release. * * Shutdown and deallocate an existing socket. */ static int llc_ui_release(struct socket *sock) { struct sock *sk = sock->sk; struct llc_sock *llc; if (unlikely(sk == NULL)) goto out; sock_hold(sk); lock_sock(sk); llc = llc_sk(sk); dprintk("%s: closing local(%02X) remote(%02X)\n", __func__, llc->laddr.lsap, llc->daddr.lsap); if (!llc_send_disc(sk)) llc_ui_wait_for_disc(sk, READ_ONCE(sk->sk_rcvtimeo)); if (!sock_flag(sk, SOCK_ZAPPED)) { struct llc_sap *sap = llc->sap; /* Hold this for release_sock(), so that llc_backlog_rcv() * could still use it. */ llc_sap_hold(sap); llc_sap_remove_socket(llc->sap, sk); release_sock(sk); llc_sap_put(sap); } else { release_sock(sk); } netdev_put(llc->dev, &llc->dev_tracker); sock_put(sk); sock_orphan(sk); sock->sk = NULL; llc_sk_free(sk); out: return 0; } /** * llc_ui_autoport - provide dynamically allocate SAP number * * Provide the caller with a dynamically allocated SAP number according * to the rules that are set in this function. Returns: 0, upon failure, * SAP number otherwise. */ static int llc_ui_autoport(void) { struct llc_sap *sap; int i, tries = 0; while (tries < LLC_SAP_DYN_TRIES) { for (i = llc_ui_sap_last_autoport; i < LLC_SAP_DYN_STOP; i += 2) { sap = llc_sap_find(i); if (!sap) { llc_ui_sap_last_autoport = i + 2; goto out; } llc_sap_put(sap); } llc_ui_sap_last_autoport = LLC_SAP_DYN_START; tries++; } i = 0; out: return i; } /** * llc_ui_autobind - automatically bind a socket to a sap * @sock: socket to bind * @addr: address to connect to * * Used by llc_ui_connect and llc_ui_sendmsg when the user hasn't * specifically used llc_ui_bind to bind to an specific address/sap * * Returns: 0 upon success, negative otherwise. */ static int llc_ui_autobind(struct socket *sock, struct sockaddr_llc *addr) { struct sock *sk = sock->sk; struct llc_sock *llc = llc_sk(sk); struct net_device *dev = NULL; struct llc_sap *sap; int rc = -EINVAL; if (!sock_flag(sk, SOCK_ZAPPED)) goto out; if (!addr->sllc_arphrd) addr->sllc_arphrd = ARPHRD_ETHER; if (addr->sllc_arphrd != ARPHRD_ETHER) goto out; rc = -ENODEV; if (sk->sk_bound_dev_if) { dev = dev_get_by_index(&init_net, sk->sk_bound_dev_if); if (dev && addr->sllc_arphrd != dev->type) { dev_put(dev); dev = NULL; } } else dev = dev_getfirstbyhwtype(&init_net, addr->sllc_arphrd); if (!dev) goto out; rc = -EUSERS; llc->laddr.lsap = llc_ui_autoport(); if (!llc->laddr.lsap) goto out; rc = -EBUSY; /* some other network layer is using the sap */ sap = llc_sap_open(llc->laddr.lsap, NULL); if (!sap) goto out; /* Note: We do not expect errors from this point. */ llc->dev = dev; netdev_tracker_alloc(llc->dev, &llc->dev_tracker, GFP_KERNEL); dev = NULL; memcpy(llc->laddr.mac, llc->dev->dev_addr, IFHWADDRLEN); memcpy(&llc->addr, addr, sizeof(llc->addr)); /* assign new connection to its SAP */ llc_sap_add_socket(sap, sk); sock_reset_flag(sk, SOCK_ZAPPED); llc_sap_put(sap); rc = 0; out: dev_put(dev); return rc; } /** * llc_ui_bind - bind a socket to a specific address. * @sock: Socket to bind an address to. * @uaddr: Address the user wants the socket bound to. * @addrlen: Length of the uaddr structure. * * Bind a socket to a specific address. For llc a user is able to bind to * a specific sap only or mac + sap. * If the user desires to bind to a specific mac + sap, it is possible to * have multiple sap connections via multiple macs. * Bind and autobind for that matter must enforce the correct sap usage * otherwise all hell will break loose. * Returns: 0 upon success, negative otherwise. */ static int llc_ui_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int addrlen) { struct sockaddr_llc *addr = (struct sockaddr_llc *)uaddr; struct sock *sk = sock->sk; struct llc_sock *llc = llc_sk(sk); struct net_device *dev = NULL; struct llc_sap *sap; int rc = -EINVAL; lock_sock(sk); if (unlikely(!sock_flag(sk, SOCK_ZAPPED) || addrlen != sizeof(*addr))) goto out; rc = -EAFNOSUPPORT; if (!addr->sllc_arphrd) addr->sllc_arphrd = ARPHRD_ETHER; if (unlikely(addr->sllc_family != AF_LLC || addr->sllc_arphrd != ARPHRD_ETHER)) goto out; dprintk("%s: binding %02X\n", __func__, addr->sllc_sap); rc = -ENODEV; rcu_read_lock(); if (sk->sk_bound_dev_if) { dev = dev_get_by_index_rcu(&init_net, sk->sk_bound_dev_if); if (dev) { if (is_zero_ether_addr(addr->sllc_mac)) memcpy(addr->sllc_mac, dev->dev_addr, IFHWADDRLEN); if (addr->sllc_arphrd != dev->type || !ether_addr_equal(addr->sllc_mac, dev->dev_addr)) { rc = -EINVAL; dev = NULL; } } } else { dev = dev_getbyhwaddr_rcu(&init_net, addr->sllc_arphrd, addr->sllc_mac); } dev_hold(dev); rcu_read_unlock(); if (!dev) goto out; if (!addr->sllc_sap) { rc = -EUSERS; addr->sllc_sap = llc_ui_autoport(); if (!addr->sllc_sap) goto out; } sap = llc_sap_find(addr->sllc_sap); if (!sap) { sap = llc_sap_open(addr->sllc_sap, NULL); rc = -EBUSY; /* some other network layer is using the sap */ if (!sap) goto out; } else { struct llc_addr laddr, daddr; struct sock *ask; memset(&laddr, 0, sizeof(laddr)); memset(&daddr, 0, sizeof(daddr)); /* * FIXME: check if the address is multicast, * only SOCK_DGRAM can do this. */ memcpy(laddr.mac, addr->sllc_mac, IFHWADDRLEN); laddr.lsap = addr->sllc_sap; rc = -EADDRINUSE; /* mac + sap clash. */ ask = llc_lookup_established(sap, &daddr, &laddr, &init_net); if (ask) { sock_put(ask); goto out_put; } } /* Note: We do not expect errors from this point. */ llc->dev = dev; netdev_tracker_alloc(llc->dev, &llc->dev_tracker, GFP_KERNEL); dev = NULL; llc->laddr.lsap = addr->sllc_sap; memcpy(llc->laddr.mac, addr->sllc_mac, IFHWADDRLEN); memcpy(&llc->addr, addr, sizeof(llc->addr)); /* assign new connection to its SAP */ llc_sap_add_socket(sap, sk); sock_reset_flag(sk, SOCK_ZAPPED); rc = 0; out_put: llc_sap_put(sap); out: dev_put(dev); release_sock(sk); return rc; } /** * llc_ui_shutdown - shutdown a connect llc2 socket. * @sock: Socket to shutdown. * @how: What part of the socket to shutdown. * * Shutdown a connected llc2 socket. Currently this function only supports * shutting down both sends and receives (2), we could probably make this * function such that a user can shutdown only half the connection but not * right now. * Returns: 0 upon success, negative otherwise. */ static int llc_ui_shutdown(struct socket *sock, int how) { struct sock *sk = sock->sk; int rc = -ENOTCONN; lock_sock(sk); if (unlikely(sk->sk_state != TCP_ESTABLISHED)) goto out; rc = -EINVAL; if (how != 2) goto out; rc = llc_send_disc(sk); if (!rc) rc = llc_ui_wait_for_disc(sk, READ_ONCE(sk->sk_rcvtimeo)); /* Wake up anyone sleeping in poll */ sk->sk_state_change(sk); out: release_sock(sk); return rc; } /** * llc_ui_connect - Connect to a remote llc2 mac + sap. * @sock: Socket which will be connected to the remote destination. * @uaddr: Remote and possibly the local address of the new connection. * @addrlen: Size of uaddr structure. * @flags: Operational flags specified by the user. * * Connect to a remote llc2 mac + sap. The caller must specify the * destination mac and address to connect to. If the user hasn't previously * called bind(2) with a smac the address of the first interface of the * specified arp type will be used. * This function will autobind if user did not previously call bind. * Returns: 0 upon success, negative otherwise. */ static int llc_ui_connect(struct socket *sock, struct sockaddr_unsized *uaddr, int addrlen, int flags) { struct sock *sk = sock->sk; struct llc_sock *llc = llc_sk(sk); struct sockaddr_llc *addr = (struct sockaddr_llc *)uaddr; int rc = -EINVAL; lock_sock(sk); if (unlikely(addrlen != sizeof(*addr))) goto out; rc = -EAFNOSUPPORT; if (unlikely(addr->sllc_family != AF_LLC)) goto out; if (unlikely(sk->sk_type != SOCK_STREAM)) goto out; rc = -EALREADY; if (unlikely(sock->state == SS_CONNECTING)) goto out; /* bind connection to sap if user hasn't done it. */ if (sock_flag(sk, SOCK_ZAPPED)) { /* bind to sap with null dev, exclusive */ rc = llc_ui_autobind(sock, addr); if (rc) goto out; } llc->daddr.lsap = addr->sllc_sap; memcpy(llc->daddr.mac, addr->sllc_mac, IFHWADDRLEN); sock->state = SS_CONNECTING; sk->sk_state = TCP_SYN_SENT; llc->link = llc_ui_next_link_no(llc->sap->laddr.lsap); rc = llc_establish_connection(sk, llc->dev->dev_addr, addr->sllc_mac, addr->sllc_sap); if (rc) { dprintk("%s: llc_ui_send_conn failed :-(\n", __func__); sock->state = SS_UNCONNECTED; sk->sk_state = TCP_CLOSE; goto out; } if (sk->sk_state == TCP_SYN_SENT) { const long timeo = sock_sndtimeo(sk, flags & O_NONBLOCK); if (!timeo || !llc_ui_wait_for_conn(sk, timeo)) { rc = -EINPROGRESS; goto out; } rc = sock_intr_errno(timeo); if (signal_pending(current)) goto out; } if (sk->sk_state == TCP_CLOSE) goto sock_error; sock->state = SS_CONNECTED; rc = 0; out: release_sock(sk); return rc; sock_error: rc = sock_error(sk) ? : -ECONNABORTED; sock->state = SS_UNCONNECTED; goto out; } /** * llc_ui_listen - allow a normal socket to accept incoming connections * @sock: Socket to allow incoming connections on. * @backlog: Number of connections to queue. * * Allow a normal socket to accept incoming connections. * Returns 0 upon success, negative otherwise. */ static int llc_ui_listen(struct socket *sock, int backlog) { struct sock *sk = sock->sk; int rc = -EINVAL; lock_sock(sk); if (unlikely(sock->state != SS_UNCONNECTED)) goto out; rc = -EOPNOTSUPP; if (unlikely(sk->sk_type != SOCK_STREAM)) goto out; rc = -EAGAIN; if (sock_flag(sk, SOCK_ZAPPED)) goto out; rc = 0; if (!(unsigned int)backlog) /* BSDism */ backlog = 1; sk->sk_max_ack_backlog = backlog; if (sk->sk_state != TCP_LISTEN) { sk->sk_ack_backlog = 0; sk->sk_state = TCP_LISTEN; } sk->sk_socket->flags |= __SO_ACCEPTCON; out: release_sock(sk); return rc; } static int llc_ui_wait_for_disc(struct sock *sk, long timeout) { DEFINE_WAIT_FUNC(wait, woken_wake_function); int rc = 0; add_wait_queue(sk_sleep(sk), &wait); while (1) { if (sk_wait_event(sk, &timeout, READ_ONCE(sk->sk_state) == TCP_CLOSE, &wait)) break; rc = -ERESTARTSYS; if (signal_pending(current)) break; rc = -EAGAIN; if (!timeout) break; rc = 0; } remove_wait_queue(sk_sleep(sk), &wait); return rc; } static bool llc_ui_wait_for_conn(struct sock *sk, long timeout) { DEFINE_WAIT_FUNC(wait, woken_wake_function); add_wait_queue(sk_sleep(sk), &wait); while (1) { if (sk_wait_event(sk, &timeout, READ_ONCE(sk->sk_state) != TCP_SYN_SENT, &wait)) break; if (signal_pending(current) || !timeout) break; } remove_wait_queue(sk_sleep(sk), &wait); return timeout; } static int llc_ui_wait_for_busy_core(struct sock *sk, long timeout) { DEFINE_WAIT_FUNC(wait, woken_wake_function); struct llc_sock *llc = llc_sk(sk); int rc; add_wait_queue(sk_sleep(sk), &wait); while (1) { rc = 0; if (sk_wait_event(sk, &timeout, (READ_ONCE(sk->sk_shutdown) & RCV_SHUTDOWN) || (!llc_data_accept_state(llc->state) && !llc->remote_busy_flag && !llc->p_flag), &wait)) break; rc = -ERESTARTSYS; if (signal_pending(current)) break; rc = -EAGAIN; if (!timeout) break; } remove_wait_queue(sk_sleep(sk), &wait); return rc; } static int llc_wait_data(struct sock *sk, long timeo) { int rc; while (1) { /* * POSIX 1003.1g mandates this order. */ rc = sock_error(sk); if (rc) break; rc = 0; if (sk->sk_shutdown & RCV_SHUTDOWN) break; rc = -EAGAIN; if (!timeo) break; rc = sock_intr_errno(timeo); if (signal_pending(current)) break; rc = 0; if (sk_wait_data(sk, &timeo, NULL)) break; } return rc; } static void llc_cmsg_rcv(struct msghdr *msg, struct sk_buff *skb) { struct llc_sock *llc = llc_sk(skb->sk); if (llc->cmsg_flags & LLC_CMSG_PKTINFO) { struct llc_pktinfo info; memset(&info, 0, sizeof(info)); info.lpi_ifindex = llc_sk(skb->sk)->dev->ifindex; llc_pdu_decode_dsap(skb, &info.lpi_sap); llc_pdu_decode_da(skb, info.lpi_mac); put_cmsg(msg, SOL_LLC, LLC_OPT_PKTINFO, sizeof(info), &info); } } /** * llc_ui_accept - accept a new incoming connection. * @sock: Socket which connections arrive on. * @newsock: Socket to move incoming connection to. * @arg: User specified arguments * * Accept a new incoming connection. * Returns 0 upon success, negative otherwise. */ static int llc_ui_accept(struct socket *sock, struct socket *newsock, struct proto_accept_arg *arg) { struct sock *sk = sock->sk, *newsk; struct llc_sock *llc, *newllc; struct sk_buff *skb; int rc = -EOPNOTSUPP; dprintk("%s: accepting on %02X\n", __func__, llc_sk(sk)->laddr.lsap); lock_sock(sk); if (unlikely(sk->sk_type != SOCK_STREAM)) goto out; rc = -EINVAL; if (unlikely(sock->state != SS_UNCONNECTED || sk->sk_state != TCP_LISTEN)) goto out; /* wait for a connection to arrive. */ if (skb_queue_empty(&sk->sk_receive_queue)) { rc = llc_wait_data(sk, READ_ONCE(sk->sk_rcvtimeo)); if (rc) goto out; } dprintk("%s: got a new connection on %02X\n", __func__, llc_sk(sk)->laddr.lsap); skb = skb_dequeue(&sk->sk_receive_queue); rc = -EINVAL; if (!skb->sk) goto frees; rc = 0; newsk = skb->sk; /* attach connection to a new socket. */ llc_ui_sk_init(newsock, newsk); sock_reset_flag(newsk, SOCK_ZAPPED); newsk->sk_state = TCP_ESTABLISHED; newsock->state = SS_CONNECTED; llc = llc_sk(sk); newllc = llc_sk(newsk); memcpy(&newllc->addr, &llc->addr, sizeof(newllc->addr)); newllc->link = llc_ui_next_link_no(newllc->laddr.lsap); /* put original socket back into a clean listen state. */ sk->sk_state = TCP_LISTEN; sk_acceptq_removed(sk); dprintk("%s: ok success on %02X, client on %02X\n", __func__, llc_sk(sk)->addr.sllc_sap, newllc->daddr.lsap); frees: kfree_skb(skb); out: release_sock(sk); return rc; } /** * llc_ui_recvmsg - copy received data to the socket user. * @sock: Socket to copy data from. * @msg: Various user space related information. * @len: Size of user buffer. * @flags: User specified flags. * * Copy received data to the socket user. * Returns non-negative upon success, negative otherwise. */ static int llc_ui_recvmsg(struct socket *sock, struct msghdr *msg, size_t len, int flags) { DECLARE_SOCKADDR(struct sockaddr_llc *, uaddr, msg->msg_name); const int nonblock = flags & MSG_DONTWAIT; struct sk_buff *skb = NULL; struct sock *sk = sock->sk; struct llc_sock *llc = llc_sk(sk); size_t copied = 0; u32 peek_seq = 0; u32 *seq, skb_len; unsigned long used; int target; /* Read at least this many bytes */ long timeo; lock_sock(sk); copied = -ENOTCONN; if (unlikely(sk->sk_type == SOCK_STREAM && sk->sk_state == TCP_LISTEN)) goto out; timeo = sock_rcvtimeo(sk, nonblock); seq = &llc->copied_seq; if (flags & MSG_PEEK) { peek_seq = llc->copied_seq; seq = &peek_seq; } target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); copied = 0; do { u32 offset; /* * We need to check signals first, to get correct SIGURG * handling. FIXME: Need to check this doesn't impact 1003.1g * and move it down to the bottom of the loop */ if (signal_pending(current)) { if (copied) break; copied = timeo ? sock_intr_errno(timeo) : -EAGAIN; break; } /* Next get a buffer. */ skb = skb_peek(&sk->sk_receive_queue); if (skb) { offset = *seq; goto found_ok_skb; } /* Well, if we have backlog, try to process it now yet. */ if (copied >= target && !READ_ONCE(sk->sk_backlog.tail)) break; if (copied) { if (sk->sk_err || sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN) || !timeo || (flags & MSG_PEEK)) break; } else { if (sock_flag(sk, SOCK_DONE)) break; if (sk->sk_err) { copied = sock_error(sk); break; } if (sk->sk_shutdown & RCV_SHUTDOWN) break; if (sk->sk_type == SOCK_STREAM && sk->sk_state == TCP_CLOSE) { if (!sock_flag(sk, SOCK_DONE)) { /* * This occurs when user tries to read * from never connected socket. */ copied = -ENOTCONN; break; } break; } if (!timeo) { copied = -EAGAIN; break; } } if (copied >= target) { /* Do not sleep, just process backlog. */ release_sock(sk); lock_sock(sk); } else sk_wait_data(sk, &timeo, NULL); if ((flags & MSG_PEEK) && peek_seq != llc->copied_seq) { net_dbg_ratelimited("LLC(%s:%d): Application bug, race in MSG_PEEK\n", current->comm, task_pid_nr(current)); peek_seq = llc->copied_seq; } continue; found_ok_skb: skb_len = skb->len; /* Ok so how much can we use? */ used = skb->len - offset; if (len < used) used = len; if (!(flags & MSG_TRUNC)) { int rc = skb_copy_datagram_msg(skb, offset, msg, used); if (rc) { /* Exception. Bailout! */ if (!copied) copied = -EFAULT; break; } } *seq += used; copied += used; len -= used; /* For non stream protcols we get one packet per recvmsg call */ if (sk->sk_type != SOCK_STREAM) goto copy_uaddr; /* Partial read */ if (used + offset < skb_len) continue; if (!(flags & MSG_PEEK)) { skb_unlink(skb, &sk->sk_receive_queue); kfree_skb(skb); *seq = 0; } } while (len > 0); out: release_sock(sk); return copied; copy_uaddr: if (uaddr != NULL && skb != NULL) { memcpy(uaddr, llc_ui_skb_cb(skb), sizeof(*uaddr)); msg->msg_namelen = sizeof(*uaddr); } if (llc_sk(sk)->cmsg_flags) llc_cmsg_rcv(msg, skb); if (!(flags & MSG_PEEK)) { skb_unlink(skb, &sk->sk_receive_queue); kfree_skb(skb); *seq = 0; } goto out; } /** * llc_ui_sendmsg - Transmit data provided by the socket user. * @sock: Socket to transmit data from. * @msg: Various user related information. * @len: Length of data to transmit. * * Transmit data provided by the socket user. * Returns non-negative upon success, negative otherwise. */ static int llc_ui_sendmsg(struct socket *sock, struct msghdr *msg, size_t len) { DECLARE_SOCKADDR(struct sockaddr_llc *, addr, msg->msg_name); struct sock *sk = sock->sk; struct llc_sock *llc = llc_sk(sk); int flags = msg->msg_flags; int noblock = flags & MSG_DONTWAIT; int rc = -EINVAL, copied = 0, hdrlen, hh_len; struct sk_buff *skb = NULL; struct net_device *dev; size_t size = 0; dprintk("%s: sending from %02X to %02X\n", __func__, llc->laddr.lsap, llc->daddr.lsap); lock_sock(sk); if (addr) { if (msg->msg_namelen < sizeof(*addr)) goto out; } else { if (llc_ui_addr_null(&llc->addr)) goto out; addr = &llc->addr; } /* must bind connection to sap if user hasn't done it. */ if (sock_flag(sk, SOCK_ZAPPED)) { /* bind to sap with null dev, exclusive. */ rc = llc_ui_autobind(sock, addr); if (rc) goto out; } dev = llc->dev; hh_len = LL_RESERVED_SPACE(dev); hdrlen = llc_ui_header_len(sk, addr); size = hdrlen + len; size = min_t(size_t, size, READ_ONCE(dev->mtu)); copied = size - hdrlen; rc = -EINVAL; if (copied < 0) goto out; release_sock(sk); skb = sock_alloc_send_skb(sk, hh_len + size, noblock, &rc); lock_sock(sk); if (!skb) goto out; if (sock_flag(sk, SOCK_ZAPPED) || llc->dev != dev || hdrlen != llc_ui_header_len(sk, addr) || hh_len != LL_RESERVED_SPACE(dev) || size > READ_ONCE(dev->mtu)) goto out; skb->dev = dev; skb->protocol = llc_proto_type(addr->sllc_arphrd); skb_reserve(skb, hh_len + hdrlen); rc = memcpy_from_msg(skb_put(skb, copied), msg, copied); if (rc) goto out; if (sk->sk_type == SOCK_DGRAM || addr->sllc_ua) { llc_build_and_send_ui_pkt(llc->sap, skb, addr->sllc_mac, addr->sllc_sap); skb = NULL; goto out; } if (addr->sllc_test) { llc_build_and_send_test_pkt(llc->sap, skb, addr->sllc_mac, addr->sllc_sap); skb = NULL; goto out; } if (addr->sllc_xid) { llc_build_and_send_xid_pkt(llc->sap, skb, addr->sllc_mac, addr->sllc_sap); skb = NULL; goto out; } rc = -ENOPROTOOPT; if (!(sk->sk_type == SOCK_STREAM && !addr->sllc_ua)) goto out; rc = llc_ui_send_data(sk, skb, noblock); skb = NULL; out: kfree_skb(skb); if (rc) dprintk("%s: failed sending from %02X to %02X: %d\n", __func__, llc->laddr.lsap, llc->daddr.lsap, rc); release_sock(sk); return rc ? : copied; } /** * llc_ui_getname - return the address info of a socket * @sock: Socket to get address of. * @uaddr: Address structure to return information. * @peer: Does user want local or remote address information. * * Return the address information of a socket. */ static int llc_ui_getname(struct socket *sock, struct sockaddr *uaddr, int peer) { struct sockaddr_llc sllc; struct sock *sk = sock->sk; struct llc_sock *llc = llc_sk(sk); int rc = -EBADF; memset(&sllc, 0, sizeof(sllc)); lock_sock(sk); if (sock_flag(sk, SOCK_ZAPPED)) goto out; if (peer) { rc = -ENOTCONN; if (sk->sk_state != TCP_ESTABLISHED) goto out; if(llc->dev) sllc.sllc_arphrd = llc->dev->type; sllc.sllc_sap = llc->daddr.lsap; memcpy(&sllc.sllc_mac, &llc->daddr.mac, IFHWADDRLEN); } else { rc = -EINVAL; if (!llc->sap) goto out; sllc.sllc_sap = llc->sap->laddr.lsap; if (llc->dev) { sllc.sllc_arphrd = llc->dev->type; memcpy(&sllc.sllc_mac, llc->dev->dev_addr, IFHWADDRLEN); } } sllc.sllc_family = AF_LLC; memcpy(uaddr, &sllc, sizeof(sllc)); rc = sizeof(sllc); out: release_sock(sk); return rc; } /** * llc_ui_ioctl - io controls for PF_LLC * @sock: Socket to get/set info * @cmd: command * @arg: optional argument for cmd * * get/set info on llc sockets */ static int llc_ui_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg) { return -ENOIOCTLCMD; } /** * llc_ui_setsockopt - set various connection specific parameters. * @sock: Socket to set options on. * @level: Socket level user is requesting operations on. * @optname: Operation name. * @optval: User provided operation data. * @optlen: Length of optval. * * Set various connection specific parameters. */ static int llc_ui_setsockopt(struct socket *sock, int level, int optname, sockptr_t optval, unsigned int optlen) { struct sock *sk = sock->sk; struct llc_sock *llc = llc_sk(sk); unsigned int opt; int rc = -EINVAL; lock_sock(sk); if (unlikely(level != SOL_LLC || optlen != sizeof(int))) goto out; rc = copy_safe_from_sockptr(&opt, sizeof(opt), optval, optlen); if (rc) goto out; rc = -EINVAL; switch (optname) { case LLC_OPT_RETRY: if (opt > LLC_OPT_MAX_RETRY) goto out; llc->n2 = opt; break; case LLC_OPT_SIZE: if (opt > LLC_OPT_MAX_SIZE) goto out; llc->n1 = opt; break; case LLC_OPT_ACK_TMR_EXP: if (opt > LLC_OPT_MAX_ACK_TMR_EXP) goto out; llc->ack_timer.expire = opt * HZ; break; case LLC_OPT_P_TMR_EXP: if (opt > LLC_OPT_MAX_P_TMR_EXP) goto out; llc->pf_cycle_timer.expire = opt * HZ; break; case LLC_OPT_REJ_TMR_EXP: if (opt > LLC_OPT_MAX_REJ_TMR_EXP) goto out; llc->rej_sent_timer.expire = opt * HZ; break; case LLC_OPT_BUSY_TMR_EXP: if (opt > LLC_OPT_MAX_BUSY_TMR_EXP) goto out; llc->busy_state_timer.expire = opt * HZ; break; case LLC_OPT_TX_WIN: if (opt > LLC_OPT_MAX_WIN) goto out; llc->k = opt; break; case LLC_OPT_RX_WIN: if (opt > LLC_OPT_MAX_WIN) goto out; llc->rw = opt; break; case LLC_OPT_PKTINFO: if (opt) llc->cmsg_flags |= LLC_CMSG_PKTINFO; else llc->cmsg_flags &= ~LLC_CMSG_PKTINFO; break; default: rc = -ENOPROTOOPT; goto out; } rc = 0; out: release_sock(sk); return rc; } /** * llc_ui_getsockopt - get connection specific socket info * @sock: Socket to get information from. * @level: Socket level user is requesting operations on. * @optname: Operation name. * @opt: sockopt context with iterator and length for returning data. * * Get connection specific socket information. */ static int llc_ui_getsockopt(struct socket *sock, int level, int optname, sockopt_t *opt) { struct sock *sk = sock->sk; struct llc_sock *llc = llc_sk(sk); int val = 0, len, rc = -EINVAL; lock_sock(sk); if (unlikely(level != SOL_LLC)) goto out; len = opt->optlen; if (len != sizeof(int)) goto out; switch (optname) { case LLC_OPT_RETRY: val = llc->n2; break; case LLC_OPT_SIZE: val = llc->n1; break; case LLC_OPT_ACK_TMR_EXP: val = llc->ack_timer.expire / HZ; break; case LLC_OPT_P_TMR_EXP: val = llc->pf_cycle_timer.expire / HZ; break; case LLC_OPT_REJ_TMR_EXP: val = llc->rej_sent_timer.expire / HZ; break; case LLC_OPT_BUSY_TMR_EXP: val = llc->busy_state_timer.expire / HZ; break; case LLC_OPT_TX_WIN: val = llc->k; break; case LLC_OPT_RX_WIN: val = llc->rw; break; case LLC_OPT_PKTINFO: val = (llc->cmsg_flags & LLC_CMSG_PKTINFO) != 0; break; default: rc = -ENOPROTOOPT; goto out; } rc = 0; opt->optlen = len; if (copy_to_iter(&val, len, &opt->iter_out) != len) rc = -EFAULT; out: release_sock(sk); return rc; } static const struct net_proto_family llc_ui_family_ops = { .family = PF_LLC, .create = llc_ui_create, .owner = THIS_MODULE, }; static const struct proto_ops llc_ui_ops = { .family = PF_LLC, .owner = THIS_MODULE, .release = llc_ui_release, .bind = llc_ui_bind, .connect = llc_ui_connect, .socketpair = sock_no_socketpair, .accept = llc_ui_accept, .getname = llc_ui_getname, .poll = datagram_poll, .ioctl = llc_ui_ioctl, .listen = llc_ui_listen, .shutdown = llc_ui_shutdown, .setsockopt = llc_ui_setsockopt, .getsockopt_iter = llc_ui_getsockopt, .sendmsg = llc_ui_sendmsg, .recvmsg = llc_ui_recvmsg, .mmap = sock_no_mmap, }; static const char llc_proc_err_msg[] __initconst = KERN_CRIT "LLC: Unable to register the proc_fs entries\n"; static const char llc_sysctl_err_msg[] __initconst = KERN_CRIT "LLC: Unable to register the sysctl entries\n"; static const char llc_sock_err_msg[] __initconst = KERN_CRIT "LLC: Unable to register the network family\n"; static int __init llc2_init(void) { int rc = proto_register(&llc_proto, 0); if (rc != 0) goto out; llc_build_offset_table(); llc_station_init(); llc_ui_sap_last_autoport = LLC_SAP_DYN_START; rc = llc_proc_init(); if (rc != 0) { printk(llc_proc_err_msg); goto out_station; } rc = llc_sysctl_init(); if (rc) { printk(llc_sysctl_err_msg); goto out_proc; } rc = sock_register(&llc_ui_family_ops); if (rc) { printk(llc_sock_err_msg); goto out_sysctl; } llc_add_pack(LLC_DEST_SAP, llc_sap_handler); llc_add_pack(LLC_DEST_CONN, llc_conn_handler); out: return rc; out_sysctl: llc_sysctl_exit(); out_proc: llc_proc_exit(); out_station: llc_station_exit(); proto_unregister(&llc_proto); goto out; } static void __exit llc2_exit(void) { llc_station_exit(); llc_remove_pack(LLC_DEST_SAP); llc_remove_pack(LLC_DEST_CONN); sock_unregister(PF_LLC); llc_proc_exit(); llc_sysctl_exit(); proto_unregister(&llc_proto); } module_init(llc2_init); module_exit(llc2_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Procom 1997, Jay Schullist 2001, Arnaldo C. Melo 2001-2003"); MODULE_DESCRIPTION("IEEE 802.2 PF_LLC support"); MODULE_ALIAS_NETPROTO(PF_LLC); |
| 3278 3274 3274 3286 7 7 7 7 8 8 7 7 7 7 7 7 1778 8 1778 8 1771 11 2 9 11 3274 7 7 3280 7 7 3284 7 7 7 7 7 7 8 8 2 8 8 8 8 8 3282 3290 3280 1771 7 7 7 2328 8 6 2 2328 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 | // SPDX-License-Identifier: GPL-2.0-only /* * Link physical devices with ACPI devices support * * Copyright (c) 2005 David Shaohua Li <shaohua.li@intel.com> * Copyright (c) 2005 Intel Corp. */ #define pr_fmt(fmt) "ACPI: " fmt #include <linux/acpi_iort.h> #include <linux/export.h> #include <linux/init.h> #include <linux/list.h> #include <linux/device.h> #include <linux/slab.h> #include <linux/rwsem.h> #include <linux/acpi.h> #include <linux/dma-mapping.h> #include <linux/pci.h> #include <linux/pci-acpi.h> #include <linux/platform_device.h> #include "internal.h" static LIST_HEAD(bus_type_list); static DECLARE_RWSEM(bus_type_sem); #define PHYSICAL_NODE_STRING "physical_node" #define PHYSICAL_NODE_NAME_SIZE (sizeof(PHYSICAL_NODE_STRING) + 10) int register_acpi_bus_type(struct acpi_bus_type *type) { if (acpi_disabled) return -ENODEV; if (type && type->match && type->find_companion) { down_write(&bus_type_sem); list_add_tail(&type->list, &bus_type_list); up_write(&bus_type_sem); pr_info("bus type %s registered\n", type->name); return 0; } return -ENODEV; } EXPORT_SYMBOL_GPL(register_acpi_bus_type); int unregister_acpi_bus_type(struct acpi_bus_type *type) { if (acpi_disabled) return 0; if (type) { down_write(&bus_type_sem); list_del_init(&type->list); up_write(&bus_type_sem); pr_info("bus type %s unregistered\n", type->name); return 0; } return -ENODEV; } EXPORT_SYMBOL_GPL(unregister_acpi_bus_type); static struct acpi_bus_type *acpi_get_bus_type(struct device *dev) { struct acpi_bus_type *tmp, *ret = NULL; down_read(&bus_type_sem); list_for_each_entry(tmp, &bus_type_list, list) { if (tmp->match(dev)) { ret = tmp; break; } } up_read(&bus_type_sem); return ret; } #define FIND_CHILD_MIN_SCORE 1 #define FIND_CHILD_MID_SCORE 2 #define FIND_CHILD_MAX_SCORE 3 static int match_any(struct acpi_device *adev, void *not_used) { return 1; } static bool acpi_dev_has_children(struct acpi_device *adev) { return acpi_dev_for_each_child(adev, match_any, NULL) > 0; } static int find_child_checks(struct acpi_device *adev, bool check_children) { unsigned long long sta; acpi_status status; if (check_children && !acpi_dev_has_children(adev)) return -ENODEV; status = acpi_evaluate_integer(adev->handle, "_STA", NULL, &sta); if (status == AE_NOT_FOUND) { /* * Special case: backlight device objects without _STA are * preferred to other objects with the same _ADR value, because * it is more likely that they are actually useful. */ if (adev->pnp.type.backlight) return FIND_CHILD_MID_SCORE; return FIND_CHILD_MIN_SCORE; } if (ACPI_FAILURE(status) || !(sta & ACPI_STA_DEVICE_ENABLED)) return -ENODEV; /* * If the device has a _HID returning a valid ACPI/PNP device ID, it is * better to make it look less attractive here, so that the other device * with the same _ADR value (that may not have a valid device ID) can be * matched going forward. [This means a second spec violation in a row, * so whatever we do here is best effort anyway.] */ if (adev->pnp.type.platform_id) return FIND_CHILD_MIN_SCORE; return FIND_CHILD_MAX_SCORE; } struct find_child_walk_data { struct acpi_device *adev; u64 address; int score; bool check_sta; bool check_children; }; static int check_one_child(struct acpi_device *adev, void *data) { struct find_child_walk_data *wd = data; int score; if (!adev->pnp.type.bus_address || acpi_device_adr(adev) != wd->address) return 0; if (!wd->adev) { /* * This is the first matching object, so save it. If it is not * necessary to look for any other matching objects, stop the * search. */ wd->adev = adev; return !(wd->check_sta || wd->check_children); } /* * There is more than one matching device object with the same _ADR * value. That really is unexpected, so we are kind of beyond the scope * of the spec here. We have to choose which one to return, though. * * First, get the score for the previously found object and terminate * the walk if it is maximum. */ if (!wd->score) { score = find_child_checks(wd->adev, wd->check_children); if (score == FIND_CHILD_MAX_SCORE) return 1; wd->score = score; } /* * Second, if the object that has just been found has a better score, * replace the previously found one with it and terminate the walk if * the new score is maximum. */ score = find_child_checks(adev, wd->check_children); if (score > wd->score) { wd->adev = adev; if (score == FIND_CHILD_MAX_SCORE) return 1; wd->score = score; } /* Continue, because there may be better matches. */ return 0; } static struct acpi_device *acpi_find_child(struct acpi_device *parent, u64 address, bool check_children, bool check_sta) { struct find_child_walk_data wd = { .address = address, .check_children = check_children, .check_sta = check_sta, .adev = NULL, .score = 0, }; if (parent) acpi_dev_for_each_child(parent, check_one_child, &wd); return wd.adev; } struct acpi_device *acpi_find_child_device(struct acpi_device *parent, u64 address, bool check_children) { return acpi_find_child(parent, address, check_children, true); } EXPORT_SYMBOL_GPL(acpi_find_child_device); struct acpi_device *acpi_find_child_by_adr(struct acpi_device *adev, acpi_bus_address adr) { return acpi_find_child(adev, adr, false, false); } EXPORT_SYMBOL_GPL(acpi_find_child_by_adr); static void acpi_physnode_link_name(char *buf, unsigned int node_id) { if (node_id > 0) snprintf(buf, PHYSICAL_NODE_NAME_SIZE, PHYSICAL_NODE_STRING "%u", node_id); else strcpy(buf, PHYSICAL_NODE_STRING); } int acpi_bind_one(struct device *dev, struct acpi_device *acpi_dev) { struct acpi_device_physical_node *physical_node, *pn; char physical_node_name[PHYSICAL_NODE_NAME_SIZE]; struct list_head *physnode_list; unsigned int node_id; int retval = -EINVAL; if (has_acpi_companion(dev)) { if (acpi_dev) { dev_warn(dev, "ACPI companion already set\n"); return -EINVAL; } else { acpi_dev = ACPI_COMPANION(dev); } } if (!acpi_dev) return -EINVAL; acpi_dev_get(acpi_dev); get_device(dev); physical_node = kzalloc_obj(*physical_node); if (!physical_node) { retval = -ENOMEM; goto err; } mutex_lock(&acpi_dev->physical_node_lock); /* * Keep the list sorted by node_id so that the IDs of removed nodes can * be recycled easily. */ physnode_list = &acpi_dev->physical_node_list; node_id = 0; list_for_each_entry(pn, &acpi_dev->physical_node_list, node) { /* Sanity check. */ if (pn->dev == dev) { mutex_unlock(&acpi_dev->physical_node_lock); dev_warn(dev, "Already associated with ACPI node\n"); kfree(physical_node); if (ACPI_COMPANION(dev) != acpi_dev) goto err; put_device(dev); acpi_dev_put(acpi_dev); return 0; } if (pn->node_id == node_id) { physnode_list = &pn->node; node_id++; } } physical_node->node_id = node_id; physical_node->dev = dev; list_add(&physical_node->node, physnode_list); acpi_dev->physical_node_count++; if (!has_acpi_companion(dev)) ACPI_COMPANION_SET(dev, acpi_dev); acpi_physnode_link_name(physical_node_name, node_id); retval = sysfs_create_link(&acpi_dev->dev.kobj, &dev->kobj, physical_node_name); if (retval) dev_err(&acpi_dev->dev, "Failed to create link %s (%d)\n", physical_node_name, retval); retval = sysfs_create_link(&dev->kobj, &acpi_dev->dev.kobj, "firmware_node"); if (retval) dev_err(dev, "Failed to create link firmware_node (%d)\n", retval); mutex_unlock(&acpi_dev->physical_node_lock); if (acpi_dev->wakeup.flags.valid) device_set_wakeup_capable(dev, true); return 0; err: ACPI_COMPANION_SET(dev, NULL); put_device(dev); acpi_dev_put(acpi_dev); return retval; } EXPORT_SYMBOL_GPL(acpi_bind_one); int acpi_unbind_one(struct device *dev) { struct acpi_device *acpi_dev = ACPI_COMPANION(dev); struct acpi_device_physical_node *entry; if (!acpi_dev) return 0; mutex_lock(&acpi_dev->physical_node_lock); list_for_each_entry(entry, &acpi_dev->physical_node_list, node) if (entry->dev == dev) { char physnode_name[PHYSICAL_NODE_NAME_SIZE]; list_del(&entry->node); acpi_dev->physical_node_count--; acpi_physnode_link_name(physnode_name, entry->node_id); sysfs_remove_link(&acpi_dev->dev.kobj, physnode_name); sysfs_remove_link(&dev->kobj, "firmware_node"); ACPI_COMPANION_SET(dev, NULL); /* Drop references taken by acpi_bind_one(). */ put_device(dev); acpi_dev_put(acpi_dev); kfree(entry); break; } mutex_unlock(&acpi_dev->physical_node_lock); return 0; } EXPORT_SYMBOL_GPL(acpi_unbind_one); void acpi_device_notify(struct device *dev) { struct acpi_device *adev; int ret; ret = acpi_bind_one(dev, NULL); if (ret) { struct acpi_bus_type *type = acpi_get_bus_type(dev); if (!type) goto err; adev = type->find_companion(dev); if (!adev) { dev_dbg(dev, "ACPI companion not found\n"); goto err; } ret = acpi_bind_one(dev, adev); if (ret) goto err; if (type->setup) { type->setup(dev); goto done; } } else { adev = ACPI_COMPANION(dev); if (dev_is_pci(dev)) { pci_acpi_setup(dev, adev); goto done; } else if (dev_is_platform(dev)) { acpi_configure_pmsi_domain(dev); } } if (adev->handler && adev->handler->bind) adev->handler->bind(dev); done: acpi_handle_debug(ACPI_HANDLE(dev), "Bound to device %s\n", dev_name(dev)); return; err: dev_dbg(dev, "No ACPI support\n"); } void acpi_device_notify_remove(struct device *dev) { struct acpi_device *adev = ACPI_COMPANION(dev); if (!adev) return; if (dev_is_pci(dev)) pci_acpi_cleanup(dev, adev); else if (adev->handler && adev->handler->unbind) adev->handler->unbind(dev); acpi_unbind_one(dev); } |
| 1 1 4 4 4 3 2 3 3 3 3 3 2 2 2 2 2 2 2 2 2 2 2 2 1 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 1 2 2 2 2 2 1 1 1 1 8 6 4 2 8 2 1 1 8 8 8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 | // SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2000-2005 Silicon Graphics, Inc. * Copyright (c) 2013 Red Hat, Inc. * All Rights Reserved. */ #include "xfs_platform.h" #include "xfs_fs.h" #include "xfs_shared.h" #include "xfs_format.h" #include "xfs_log_format.h" #include "xfs_trans_resv.h" #include "xfs_mount.h" #include "xfs_da_format.h" #include "xfs_inode.h" #include "xfs_trans.h" #include "xfs_bmap.h" #include "xfs_da_btree.h" #include "xfs_attr.h" #include "xfs_attr_sf.h" #include "xfs_attr_leaf.h" #include "xfs_error.h" #include "xfs_trace.h" #include "xfs_dir2.h" #include "xfs_health.h" STATIC int xfs_attr_shortform_compare(const void *a, const void *b) { xfs_attr_sf_sort_t *sa, *sb; sa = (xfs_attr_sf_sort_t *)a; sb = (xfs_attr_sf_sort_t *)b; if (sa->hash < sb->hash) { return -1; } else if (sa->hash > sb->hash) { return 1; } else { return sa->entno - sb->entno; } } #define XFS_ISRESET_CURSOR(cursor) \ (!((cursor)->initted) && !((cursor)->hashval) && \ !((cursor)->blkno) && !((cursor)->offset)) /* * Copy out entries of shortform attribute lists for attr_list(). * Shortform attribute lists are not stored in hashval sorted order. * If the output buffer is not large enough to hold them all, then * we have to calculate each entries' hashvalue and sort them before * we can begin returning them to the user. */ static int xfs_attr_shortform_list( struct xfs_attr_list_context *context) { struct xfs_attrlist_cursor_kern *cursor = &context->cursor; struct xfs_inode *dp = context->dp; struct xfs_attr_sf_sort *sbuf, *sbp; struct xfs_attr_sf_hdr *sf = dp->i_af.if_data; struct xfs_attr_sf_entry *sfe; int sbsize, nsbuf, count, i; int error = 0; ASSERT(sf != NULL); if (!sf->count) return 0; trace_xfs_attr_list_sf(context); /* * If the buffer is large enough and the cursor is at the start, * do not bother with sorting since we will return everything in * one buffer and another call using the cursor won't need to be * made. * Note the generous fudge factor of 16 overhead bytes per entry. * If bufsize is zero then put_listent must be a search function * and can just scan through what we have. */ if (context->bufsize == 0 || (XFS_ISRESET_CURSOR(cursor) && (dp->i_af.if_bytes + sf->count * 16) < context->bufsize)) { for (i = 0, sfe = xfs_attr_sf_firstentry(sf); i < sf->count; i++) { if (XFS_IS_CORRUPT(context->dp->i_mount, !xfs_attr_namecheck(sfe->flags, sfe->nameval, sfe->namelen))) { xfs_dirattr_mark_sick(context->dp, XFS_ATTR_FORK); return -EFSCORRUPTED; } context->put_listent(context, sfe->flags, sfe->nameval, (int)sfe->namelen, &sfe->nameval[sfe->namelen], (int)sfe->valuelen); /* * Either search callback finished early or * didn't fit it all in the buffer after all. */ if (context->seen_enough) break; sfe = xfs_attr_sf_nextentry(sfe); } trace_xfs_attr_list_sf_all(context); return 0; } /* do no more for a search callback */ if (context->bufsize == 0) return 0; /* * It didn't all fit, so we have to sort everything on hashval. */ sbsize = sf->count * sizeof(*sbuf); sbp = sbuf = kmalloc(sbsize, GFP_KERNEL | __GFP_NOLOCKDEP | __GFP_NOFAIL); /* * Scan the attribute list for the rest of the entries, storing * the relevant info from only those that match into a buffer. */ nsbuf = 0; for (i = 0, sfe = xfs_attr_sf_firstentry(sf); i < sf->count; i++) { if (unlikely( ((char *)sfe < (char *)sf) || ((char *)sfe >= ((char *)sf + dp->i_af.if_bytes)) || !xfs_attr_check_namespace(sfe->flags))) { XFS_CORRUPTION_ERROR("xfs_attr_shortform_list", XFS_ERRLEVEL_LOW, context->dp->i_mount, sfe, sizeof(*sfe)); kfree(sbuf); xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); return -EFSCORRUPTED; } sbp->entno = i; sbp->name = sfe->nameval; sbp->namelen = sfe->namelen; /* These are bytes, and both on-disk, don't endian-flip */ sbp->value = &sfe->nameval[sfe->namelen]; sbp->valuelen = sfe->valuelen; sbp->flags = sfe->flags; sbp->hash = xfs_attr_hashval(dp->i_mount, sfe->flags, sfe->nameval, sfe->namelen, sfe->nameval + sfe->namelen, sfe->valuelen); sfe = xfs_attr_sf_nextentry(sfe); sbp++; nsbuf++; } /* * Sort the entries on hash then entno. */ xfs_sort(sbuf, nsbuf, sizeof(*sbuf), xfs_attr_shortform_compare); /* * Re-find our place IN THE SORTED LIST. */ count = 0; cursor->initted = 1; cursor->blkno = 0; for (sbp = sbuf, i = 0; i < nsbuf; i++, sbp++) { if (sbp->hash == cursor->hashval) { if (cursor->offset == count) { break; } count++; } else if (sbp->hash > cursor->hashval) { break; } } if (i == nsbuf) goto out; /* * Loop putting entries into the user buffer. */ for ( ; i < nsbuf; i++, sbp++) { if (cursor->hashval != sbp->hash) { cursor->hashval = sbp->hash; cursor->offset = 0; } if (XFS_IS_CORRUPT(context->dp->i_mount, !xfs_attr_namecheck(sbp->flags, sbp->name, sbp->namelen))) { xfs_dirattr_mark_sick(context->dp, XFS_ATTR_FORK); error = -EFSCORRUPTED; goto out; } context->put_listent(context, sbp->flags, sbp->name, sbp->namelen, sbp->value, sbp->valuelen); if (context->seen_enough) break; cursor->offset++; } out: kfree(sbuf); return error; } /* * We didn't find the block & hash mentioned in the cursor state, so * walk down the attr btree looking for the hash. */ STATIC int xfs_attr_node_list_lookup( struct xfs_attr_list_context *context, struct xfs_attrlist_cursor_kern *cursor, struct xfs_buf **pbp) { struct xfs_da3_icnode_hdr nodehdr; struct xfs_da_intnode *node; struct xfs_da_node_entry *btree; struct xfs_inode *dp = context->dp; struct xfs_mount *mp = dp->i_mount; struct xfs_trans *tp = context->tp; struct xfs_buf *bp; xfs_failaddr_t fa; int i; int error = 0; unsigned int expected_level = 0; uint16_t magic; ASSERT(*pbp == NULL); cursor->blkno = 0; for (;;) { error = xfs_da3_node_read(tp, dp, cursor->blkno, &bp, XFS_ATTR_FORK); if (error) return error; node = bp->b_addr; magic = be16_to_cpu(node->hdr.info.magic); if (magic == XFS_ATTR_LEAF_MAGIC || magic == XFS_ATTR3_LEAF_MAGIC) break; if (magic != XFS_DA_NODE_MAGIC && magic != XFS_DA3_NODE_MAGIC) { XFS_CORRUPTION_ERROR(__func__, XFS_ERRLEVEL_LOW, mp, node, sizeof(*node)); goto out_corruptbuf; } fa = xfs_da3_node_header_check(bp, I_INO(dp)); if (fa) goto out_corruptbuf; xfs_da3_node_hdr_from_disk(mp, &nodehdr, node); /* Tree taller than we can handle; bail out! */ if (nodehdr.level >= XFS_DA_NODE_MAXDEPTH) goto out_corruptbuf; /* Check the level from the root node. */ if (cursor->blkno == 0) expected_level = nodehdr.level - 1; else if (expected_level != nodehdr.level) goto out_corruptbuf; else expected_level--; btree = nodehdr.btree; for (i = 0; i < nodehdr.count; btree++, i++) { if (cursor->hashval <= be32_to_cpu(btree->hashval)) { cursor->blkno = be32_to_cpu(btree->before); trace_xfs_attr_list_node_descend(context, btree); break; } } xfs_trans_brelse(tp, bp); if (i == nodehdr.count) return 0; /* We can't point back to the root. */ if (XFS_IS_CORRUPT(mp, cursor->blkno == 0)) { xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); return -EFSCORRUPTED; } } fa = xfs_attr3_leaf_header_check(bp, I_INO(dp)); if (fa) { __xfs_buf_mark_corrupt(bp, fa); goto out_releasebuf; } if (expected_level != 0) goto out_corruptbuf; *pbp = bp; return 0; out_corruptbuf: xfs_buf_mark_corrupt(bp); out_releasebuf: xfs_trans_brelse(tp, bp); xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); return -EFSCORRUPTED; } STATIC int xfs_attr_node_list( struct xfs_attr_list_context *context) { struct xfs_attrlist_cursor_kern *cursor = &context->cursor; struct xfs_attr3_icleaf_hdr leafhdr; struct xfs_attr_leafblock *leaf; struct xfs_da_intnode *node; struct xfs_buf *bp; struct xfs_inode *dp = context->dp; struct xfs_mount *mp = dp->i_mount; xfs_failaddr_t fa; int error = 0; trace_xfs_attr_node_list(context); cursor->initted = 1; /* * Do all sorts of validation on the passed-in cursor structure. * If anything is amiss, ignore the cursor and look up the hashval * starting from the btree root. */ bp = NULL; if (cursor->blkno > 0) { struct xfs_attr_leaf_entry *entries; error = xfs_da3_node_read(context->tp, dp, cursor->blkno, &bp, XFS_ATTR_FORK); if (xfs_metadata_is_sick(error)) xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); if (error != 0 && error != -EFSCORRUPTED) return error; if (!bp) goto need_lookup; node = bp->b_addr; switch (be16_to_cpu(node->hdr.info.magic)) { case XFS_DA_NODE_MAGIC: case XFS_DA3_NODE_MAGIC: trace_xfs_attr_list_wrong_blk(context); fa = xfs_da3_node_header_check(bp, I_INO(dp)); if (fa) { __xfs_buf_mark_corrupt(bp, fa); xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); } xfs_trans_brelse(context->tp, bp); bp = NULL; break; case XFS_ATTR_LEAF_MAGIC: case XFS_ATTR3_LEAF_MAGIC: leaf = bp->b_addr; fa = xfs_attr3_leaf_header_check(bp, I_INO(dp)); if (fa) { __xfs_buf_mark_corrupt(bp, fa); xfs_trans_brelse(context->tp, bp); xfs_dirattr_mark_sick(dp, XFS_ATTR_FORK); bp = NULL; break; } xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf); entries = xfs_attr3_leaf_entryp(leaf); if (cursor->hashval > be32_to_cpu( entries[leafhdr.count - 1].hashval)) { trace_xfs_attr_list_wrong_blk(context); xfs_trans_brelse(context->tp, bp); bp = NULL; } else if (cursor->hashval <= be32_to_cpu( entries[0].hashval)) { trace_xfs_attr_list_wrong_blk(context); xfs_trans_brelse(context->tp, bp); bp = NULL; } break; default: trace_xfs_attr_list_wrong_blk(context); xfs_trans_brelse(context->tp, bp); bp = NULL; } } /* * We did not find what we expected given the cursor's contents, * so we start from the top and work down based on the hash value. * Note that start of node block is same as start of leaf block. */ if (bp == NULL) { need_lookup: error = xfs_attr_node_list_lookup(context, cursor, &bp); if (error || !bp) return error; } ASSERT(bp != NULL); /* * Roll upward through the blocks, processing each leaf block in * order. As long as there is space in the result buffer, keep * adding the information. */ for (;;) { leaf = bp->b_addr; error = xfs_attr3_leaf_list_int(bp, context); if (error) break; xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &leafhdr, leaf); if (context->seen_enough || leafhdr.forw == 0) break; cursor->blkno = leafhdr.forw; xfs_trans_brelse(context->tp, bp); error = xfs_attr3_leaf_read(context->tp, dp, I_INO(dp), cursor->blkno, &bp); if (error) return error; } xfs_trans_brelse(context->tp, bp); return error; } /* * Copy out attribute list entries for attr_list(), for leaf attribute lists. */ int xfs_attr3_leaf_list_int( struct xfs_buf *bp, struct xfs_attr_list_context *context) { struct xfs_attrlist_cursor_kern *cursor = &context->cursor; struct xfs_attr_leafblock *leaf; struct xfs_attr3_icleaf_hdr ichdr; struct xfs_attr_leaf_entry *entries; struct xfs_attr_leaf_entry *entry; int i; struct xfs_mount *mp = context->dp->i_mount; trace_xfs_attr_list_leaf(context); leaf = bp->b_addr; xfs_attr3_leaf_hdr_from_disk(mp->m_attr_geo, &ichdr, leaf); entries = xfs_attr3_leaf_entryp(leaf); cursor->initted = 1; /* * Re-find our place in the leaf block if this is a new syscall. */ if (context->resynch) { entry = &entries[0]; for (i = 0; i < ichdr.count; entry++, i++) { if (be32_to_cpu(entry->hashval) == cursor->hashval) { if (cursor->offset == context->dupcnt) { context->dupcnt = 0; break; } context->dupcnt++; } else if (be32_to_cpu(entry->hashval) > cursor->hashval) { context->dupcnt = 0; break; } } if (i == ichdr.count) { trace_xfs_attr_list_notfound(context); return 0; } } else { entry = &entries[0]; i = 0; } context->resynch = 0; /* * We have found our place, start copying out the new attributes. */ for (; i < ichdr.count; entry++, i++) { char *name; void *value; int namelen, valuelen; if (be32_to_cpu(entry->hashval) != cursor->hashval) { cursor->hashval = be32_to_cpu(entry->hashval); cursor->offset = 0; } if ((entry->flags & XFS_ATTR_INCOMPLETE) && !context->allow_incomplete) continue; if (entry->flags & XFS_ATTR_LOCAL) { xfs_attr_leaf_name_local_t *name_loc; name_loc = xfs_attr3_leaf_name_local(leaf, i); name = name_loc->nameval; namelen = name_loc->namelen; value = &name_loc->nameval[name_loc->namelen]; valuelen = be16_to_cpu(name_loc->valuelen); } else { xfs_attr_leaf_name_remote_t *name_rmt; name_rmt = xfs_attr3_leaf_name_remote(leaf, i); name = name_rmt->name; namelen = name_rmt->namelen; value = NULL; valuelen = be32_to_cpu(name_rmt->valuelen); } if (XFS_IS_CORRUPT(context->dp->i_mount, !xfs_attr_namecheck(entry->flags, name, namelen))) { xfs_dirattr_mark_sick(context->dp, XFS_ATTR_FORK); return -EFSCORRUPTED; } context->put_listent(context, entry->flags, name, namelen, value, valuelen); if (context->seen_enough) break; cursor->offset++; } trace_xfs_attr_list_leaf_end(context); return 0; } /* * Copy out attribute entries for attr_list(), for leaf attribute lists. */ STATIC int xfs_attr_leaf_list( struct xfs_attr_list_context *context) { struct xfs_buf *bp; int error; trace_xfs_attr_leaf_list(context); context->cursor.blkno = 0; error = xfs_attr3_leaf_read(context->tp, context->dp, I_INO(context->dp), 0, &bp); if (error) return error; error = xfs_attr3_leaf_list_int(bp, context); xfs_trans_brelse(context->tp, bp); return error; } int xfs_attr_list_ilocked( struct xfs_attr_list_context *context) { struct xfs_inode *dp = context->dp; int error; xfs_assert_ilocked(dp, XFS_ILOCK_SHARED | XFS_ILOCK_EXCL); /* * Decide on what work routines to call based on the inode size. */ if (!xfs_inode_hasattr(dp)) return 0; if (dp->i_af.if_format == XFS_DINODE_FMT_LOCAL) return xfs_attr_shortform_list(context); /* Prerequisite for xfs_attr_is_leaf */ error = xfs_iread_extents(NULL, dp, XFS_ATTR_FORK); if (error) return error; if (xfs_attr_is_leaf(dp)) return xfs_attr_leaf_list(context); return xfs_attr_node_list(context); } int xfs_attr_list( struct xfs_attr_list_context *context) { struct xfs_inode *dp = context->dp; uint lock_mode; int error; XFS_STATS_INC(dp->i_mount, xs_attr_list); if (xfs_is_shutdown(dp->i_mount)) return -EIO; lock_mode = xfs_ilock_attr_map_shared(dp); error = xfs_attr_list_ilocked(context); xfs_iunlock(dp, lock_mode); return error; } |
| 70 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 | /* SPDX-License-Identifier: GPL-2.0-or-later */ /* * NetLabel NETLINK Interface * * This file defines the NETLINK interface for the NetLabel system. The * NetLabel system manages static and dynamic label mappings for network * protocols such as CIPSO and RIPSO. * * Author: Paul Moore <paul@paul-moore.com> */ /* * (c) Copyright Hewlett-Packard Development Company, L.P., 2006 */ #ifndef _NETLABEL_USER_H #define _NETLABEL_USER_H #include <linux/types.h> #include <linux/skbuff.h> #include <linux/capability.h> #include <linux/audit.h> #include <net/netlink.h> #include <net/genetlink.h> #include <net/netlabel.h> /* NetLabel NETLINK helper functions */ /** * netlbl_netlink_auditinfo - Fetch the audit information from a NETLINK msg * @audit_info: NetLabel audit information */ static inline void netlbl_netlink_auditinfo(struct netlbl_audit *audit_info) { security_current_getlsmprop_subj(&audit_info->prop); audit_info->loginuid = audit_get_loginuid(current); audit_info->sessionid = audit_get_sessionid(current); } /* NetLabel NETLINK I/O functions */ int netlbl_netlink_init(void); /* NetLabel Audit Functions */ struct audit_buffer *netlbl_audit_start_common(int type, struct netlbl_audit *audit_info); #endif |
| 2 2 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 | /* SPDX-License-Identifier: GPL-2.0 */ /* * Definitions for the NVM Express interface * Copyright (c) 2011-2014, Intel Corporation. */ #ifndef _LINUX_NVME_H #define _LINUX_NVME_H #include <linux/bits.h> #include <linux/types.h> #include <linux/uuid.h> /* NQN names in commands fields specified one size */ #define NVMF_NQN_FIELD_LEN 256 /* However the max length of a qualified name is another size */ #define NVMF_NQN_SIZE 223 #define NVMF_TRSVCID_SIZE 32 #define NVMF_TRADDR_SIZE 256 #define NVMF_TSAS_SIZE 256 #define NVME_DISC_SUBSYS_NAME "nqn.2014-08.org.nvmexpress.discovery" #define NVME_NSID_ALL 0xffffffff /* Special NSSR value, 'NVMe' */ #define NVME_SUBSYS_RESET 0x4E564D65 enum nvme_subsys_type { /* Referral to another discovery type target subsystem */ NVME_NQN_DISC = 1, /* NVME type target subsystem */ NVME_NQN_NVME = 2, /* Current discovery type target subsystem */ NVME_NQN_CURR = 3, }; enum nvme_ctrl_type { NVME_CTRL_IO = 1, /* I/O controller */ NVME_CTRL_DISC = 2, /* Discovery controller */ NVME_CTRL_ADMIN = 3, /* Administrative controller */ }; enum nvme_dctype { NVME_DCTYPE_NOT_REPORTED = 0, NVME_DCTYPE_DDC = 1, /* Direct Discovery Controller */ NVME_DCTYPE_CDC = 2, /* Central Discovery Controller */ }; /* Address Family codes for Discovery Log Page entry ADRFAM field */ enum { NVMF_ADDR_FAMILY_PCI = 0, /* PCIe */ NVMF_ADDR_FAMILY_IP4 = 1, /* IP4 */ NVMF_ADDR_FAMILY_IP6 = 2, /* IP6 */ NVMF_ADDR_FAMILY_IB = 3, /* InfiniBand */ NVMF_ADDR_FAMILY_FC = 4, /* Fibre Channel */ NVMF_ADDR_FAMILY_LOOP = 254, /* Reserved for host usage */ NVMF_ADDR_FAMILY_MAX, }; /* Transport Type codes for Discovery Log Page entry TRTYPE field */ enum { NVMF_TRTYPE_PCI = 0, /* PCI */ NVMF_TRTYPE_RDMA = 1, /* RDMA */ NVMF_TRTYPE_FC = 2, /* Fibre Channel */ NVMF_TRTYPE_TCP = 3, /* TCP/IP */ NVMF_TRTYPE_LOOP = 254, /* Reserved for host usage */ NVMF_TRTYPE_MAX, }; /* Transport Requirements codes for Discovery Log Page entry TREQ field */ enum { NVMF_TREQ_NOT_SPECIFIED = 0, /* Not specified */ NVMF_TREQ_REQUIRED = 1, /* Required */ NVMF_TREQ_NOT_REQUIRED = 2, /* Not Required */ #define NVME_TREQ_SECURE_CHANNEL_MASK \ (NVMF_TREQ_REQUIRED | NVMF_TREQ_NOT_REQUIRED) NVMF_TREQ_DISABLE_SQFLOW = (1 << 2), /* Supports SQ flow control disable */ }; /* RDMA QP Service Type codes for Discovery Log Page entry TSAS * RDMA_QPTYPE field */ enum { NVMF_RDMA_QPTYPE_CONNECTED = 1, /* Reliable Connected */ NVMF_RDMA_QPTYPE_DATAGRAM = 2, /* Reliable Datagram */ NVMF_RDMA_QPTYPE_INVALID = 0xff, }; /* RDMA Provider Type codes for Discovery Log Page entry TSAS * RDMA_PRTYPE field */ enum { NVMF_RDMA_PRTYPE_NOT_SPECIFIED = 1, /* No Provider Specified */ NVMF_RDMA_PRTYPE_IB = 2, /* InfiniBand */ NVMF_RDMA_PRTYPE_ROCE = 3, /* InfiniBand RoCE */ NVMF_RDMA_PRTYPE_ROCEV2 = 4, /* InfiniBand RoCEV2 */ NVMF_RDMA_PRTYPE_IWARP = 5, /* IWARP */ }; /* RDMA Connection Management Service Type codes for Discovery Log Page * entry TSAS RDMA_CMS field */ enum { NVMF_RDMA_CMS_RDMA_CM = 1, /* Sockets based endpoint addressing */ }; /* TSAS SECTYPE for TCP transport */ enum { NVMF_TCP_SECTYPE_NONE = 0, /* No Security */ NVMF_TCP_SECTYPE_TLS12 = 1, /* TLSv1.2, NVMe-oF 1.1 and NVMe-TCP 3.6.1.1 */ NVMF_TCP_SECTYPE_TLS13 = 2, /* TLSv1.3, NVMe-oF 1.1 and NVMe-TCP 3.6.1.1 */ NVMF_TCP_SECTYPE_INVALID = 0xff, }; #define NVME_AQ_DEPTH 32 #define NVME_NR_AEN_COMMANDS 1 #define NVME_AQ_BLK_MQ_DEPTH (NVME_AQ_DEPTH - NVME_NR_AEN_COMMANDS) /* * Subtract one to leave an empty queue entry for 'Full Queue' condition. See * NVM-Express 1.2 specification, section 4.1.2. */ #define NVME_AQ_MQ_TAG_DEPTH (NVME_AQ_BLK_MQ_DEPTH - 1) enum { NVME_REG_CAP = 0x0000, /* Controller Capabilities */ NVME_REG_VS = 0x0008, /* Version */ NVME_REG_INTMS = 0x000c, /* Interrupt Mask Set */ NVME_REG_INTMC = 0x0010, /* Interrupt Mask Clear */ NVME_REG_CC = 0x0014, /* Controller Configuration */ NVME_REG_CSTS = 0x001c, /* Controller Status */ NVME_REG_NSSR = 0x0020, /* NVM Subsystem Reset */ NVME_REG_AQA = 0x0024, /* Admin Queue Attributes */ NVME_REG_ASQ = 0x0028, /* Admin SQ Base Address */ NVME_REG_ACQ = 0x0030, /* Admin CQ Base Address */ NVME_REG_CMBLOC = 0x0038, /* Controller Memory Buffer Location */ NVME_REG_CMBSZ = 0x003c, /* Controller Memory Buffer Size */ NVME_REG_BPINFO = 0x0040, /* Boot Partition Information */ NVME_REG_BPRSEL = 0x0044, /* Boot Partition Read Select */ NVME_REG_BPMBL = 0x0048, /* Boot Partition Memory Buffer * Location */ NVME_REG_CMBMSC = 0x0050, /* Controller Memory Buffer Memory * Space Control */ NVME_REG_CRTO = 0x0068, /* Controller Ready Timeouts */ NVME_REG_PMRCAP = 0x0e00, /* Persistent Memory Capabilities */ NVME_REG_PMRCTL = 0x0e04, /* Persistent Memory Region Control */ NVME_REG_PMRSTS = 0x0e08, /* Persistent Memory Region Status */ NVME_REG_PMREBS = 0x0e0c, /* Persistent Memory Region Elasticity * Buffer Size */ NVME_REG_PMRSWTP = 0x0e10, /* Persistent Memory Region Sustained * Write Throughput */ NVME_REG_DBS = 0x1000, /* SQ 0 Tail Doorbell */ }; #define NVME_CAP_MQES(cap) ((cap) & 0xffff) #define NVME_CAP_TIMEOUT(cap) (((cap) >> 24) & 0xff) #define NVME_CAP_STRIDE(cap) (((cap) >> 32) & 0xf) #define NVME_CAP_NSSRC(cap) (((cap) >> 36) & 0x1) #define NVME_CAP_CSS(cap) (((cap) >> 37) & 0xff) #define NVME_CAP_MPSMIN(cap) (((cap) >> 48) & 0xf) #define NVME_CAP_MPSMAX(cap) (((cap) >> 52) & 0xf) #define NVME_CAP_CMBS(cap) (((cap) >> 57) & 0x1) #define NVME_CMB_BIR(cmbloc) ((cmbloc) & 0x7) #define NVME_CMB_OFST(cmbloc) (((cmbloc) >> 12) & 0xfffff) #define NVME_CRTO_CRIMT(crto) ((crto) >> 16) #define NVME_CRTO_CRWMT(crto) ((crto) & 0xffff) enum { NVME_CMBSZ_SQS = 1 << 0, NVME_CMBSZ_CQS = 1 << 1, NVME_CMBSZ_LISTS = 1 << 2, NVME_CMBSZ_RDS = 1 << 3, NVME_CMBSZ_WDS = 1 << 4, NVME_CMBSZ_SZ_SHIFT = 12, NVME_CMBSZ_SZ_MASK = 0xfffff, NVME_CMBSZ_SZU_SHIFT = 8, NVME_CMBSZ_SZU_MASK = 0xf, }; /* * Submission and Completion Queue Entry Sizes for the NVM command set. * (In bytes and specified as a power of two (2^n)). */ #define NVME_ADM_SQES 6 #define NVME_NVM_IOSQES 6 #define NVME_NVM_IOCQES 4 /* * Controller Configuration (CC) register (Offset 14h) */ enum { /* Enable (EN): bit 0 */ NVME_CC_ENABLE = 1 << 0, NVME_CC_EN_SHIFT = 0, /* Bits 03:01 are reserved (NVMe Base Specification rev 2.1) */ /* I/O Command Set Selected (CSS): bits 06:04 */ NVME_CC_CSS_SHIFT = 4, NVME_CC_CSS_MASK = 7 << NVME_CC_CSS_SHIFT, NVME_CC_CSS_NVM = 0 << NVME_CC_CSS_SHIFT, NVME_CC_CSS_CSI = 6 << NVME_CC_CSS_SHIFT, /* Memory Page Size (MPS): bits 10:07 */ NVME_CC_MPS_SHIFT = 7, NVME_CC_MPS_MASK = 0xf << NVME_CC_MPS_SHIFT, /* Arbitration Mechanism Selected (AMS): bits 13:11 */ NVME_CC_AMS_SHIFT = 11, NVME_CC_AMS_MASK = 7 << NVME_CC_AMS_SHIFT, NVME_CC_AMS_RR = 0 << NVME_CC_AMS_SHIFT, NVME_CC_AMS_WRRU = 1 << NVME_CC_AMS_SHIFT, NVME_CC_AMS_VS = 7 << NVME_CC_AMS_SHIFT, /* Shutdown Notification (SHN): bits 15:14 */ NVME_CC_SHN_SHIFT = 14, NVME_CC_SHN_MASK = 3 << NVME_CC_SHN_SHIFT, NVME_CC_SHN_NONE = 0 << NVME_CC_SHN_SHIFT, NVME_CC_SHN_NORMAL = 1 << NVME_CC_SHN_SHIFT, NVME_CC_SHN_ABRUPT = 2 << NVME_CC_SHN_SHIFT, /* I/O Submission Queue Entry Size (IOSQES): bits 19:16 */ NVME_CC_IOSQES_SHIFT = 16, NVME_CC_IOSQES_MASK = 0xf << NVME_CC_IOSQES_SHIFT, NVME_CC_IOSQES = NVME_NVM_IOSQES << NVME_CC_IOSQES_SHIFT, /* I/O Completion Queue Entry Size (IOCQES): bits 23:20 */ NVME_CC_IOCQES_SHIFT = 20, NVME_CC_IOCQES_MASK = 0xf << NVME_CC_IOCQES_SHIFT, NVME_CC_IOCQES = NVME_NVM_IOCQES << NVME_CC_IOCQES_SHIFT, /* Controller Ready Independent of Media Enable (CRIME): bit 24 */ NVME_CC_CRIME = 1 << 24, /* Bits 25:31 are reserved (NVMe Base Specification rev 2.1) */ }; enum { NVME_CSTS_RDY = 1 << 0, NVME_CSTS_CFS = 1 << 1, NVME_CSTS_NSSRO = 1 << 4, NVME_CSTS_PP = 1 << 5, NVME_CSTS_SHST_NORMAL = 0 << 2, NVME_CSTS_SHST_OCCUR = 1 << 2, NVME_CSTS_SHST_CMPLT = 2 << 2, NVME_CSTS_SHST_MASK = 3 << 2, }; enum { NVME_CMBMSC_CRE = 1 << 0, NVME_CMBMSC_CMSE = 1 << 1, }; enum { NVME_CAP_CSS_NVM = 1 << 0, NVME_CAP_CSS_CSI = 1 << 6, }; enum { NVME_CAP_CRMS_CRWMS = 1ULL << 59, NVME_CAP_CRMS_CRIMS = 1ULL << 60, }; struct nvme_id_power_state { __le16 max_power; /* centiwatts */ __u8 rsvd2; __u8 flags; __le32 entry_lat; /* microseconds */ __le32 exit_lat; /* microseconds */ __u8 read_tput; __u8 read_lat; __u8 write_tput; __u8 write_lat; __le16 idle_power; __u8 idle_scale; __u8 rsvd19; __le16 active_power; __u8 active_work_scale; __u8 rsvd23[9]; }; enum { NVME_PS_FLAGS_MAX_POWER_SCALE = 1 << 0, NVME_PS_FLAGS_NON_OP_STATE = 1 << 1, }; enum nvme_ctrl_attr { NVME_CTRL_ATTR_HID_128_BIT = (1 << 0), NVME_CTRL_ATTR_TBKAS = (1 << 6), NVME_CTRL_ATTR_ELBAS = (1 << 15), NVME_CTRL_ATTR_RHII = (1 << 18), NVME_CTRL_ATTR_FDPS = (1 << 19), }; struct nvme_id_ctrl { __le16 vid; __le16 ssvid; char sn[20]; char mn[40]; char fr[8]; __u8 rab; __u8 ieee[3]; __u8 cmic; __u8 mdts; __le16 cntlid; __le32 ver; __le32 rtd3r; __le32 rtd3e; __le32 oaes; __le32 ctratt; __u8 rsvd100[11]; __u8 cntrltype; __u8 fguid[16]; __le16 crdt1; __le16 crdt2; __le16 crdt3; __u8 rsvd134[122]; __le16 oacs; __u8 acl; __u8 aerl; __u8 frmw; __u8 lpa; __u8 elpe; __u8 npss; __u8 avscc; __u8 apsta; __le16 wctemp; __le16 cctemp; __le16 mtfa; __le32 hmpre; __le32 hmmin; __u8 tnvmcap[16]; __u8 unvmcap[16]; __le32 rpmbs; __le16 edstt; __u8 dsto; __u8 fwug; __le16 kas; __le16 hctma; __le16 mntmt; __le16 mxtmt; __le32 sanicap; __le32 hmminds; __le16 hmmaxd; __le16 nvmsetidmax; __le16 endgidmax; __u8 anatt; __u8 anacap; __le32 anagrpmax; __le32 nanagrpid; __u8 rsvd352[160]; __u8 sqes; __u8 cqes; __le16 maxcmd; __le32 nn; __le16 oncs; __le16 fuses; __u8 fna; __u8 vwc; __le16 awun; __le16 awupf; __u8 nvscc; __u8 nwpc; __le16 acwu; __u8 rsvd534[2]; __le32 sgls; __le32 mnan; __u8 rsvd544[224]; char subnqn[256]; __u8 rsvd1024[768]; __le32 ioccsz; __le32 iorcsz; __le16 icdoff; __u8 ctrattr; __u8 msdbd; __u8 rsvd1804[2]; __u8 dctype; __u8 rsvd1807[241]; struct nvme_id_power_state psd[32]; __u8 vs[1024]; }; enum { NVME_CTRL_CMIC_MULTI_PORT = 1 << 0, NVME_CTRL_CMIC_MULTI_CTRL = 1 << 1, NVME_CTRL_CMIC_ANA = 1 << 3, NVME_CTRL_ONCS_COMPARE = 1 << 0, NVME_CTRL_ONCS_WRITE_UNCORRECTABLE = 1 << 1, NVME_CTRL_ONCS_DSM = 1 << 2, NVME_CTRL_ONCS_WRITE_ZEROES = 1 << 3, NVME_CTRL_ONCS_RESERVATIONS = 1 << 5, NVME_CTRL_ONCS_TIMESTAMP = 1 << 6, NVME_CTRL_VWC_PRESENT = 1 << 0, NVME_CTRL_OACS_SEC_SUPP = 1 << 0, NVME_CTRL_OACS_NS_MNGT_SUPP = 1 << 3, NVME_CTRL_OACS_DIRECTIVES = 1 << 5, NVME_CTRL_OACS_DBBUF_SUPP = 1 << 8, NVME_CTRL_LPA_CMD_EFFECTS_LOG = 1 << 1, NVME_CTRL_CTRATT_128_ID = 1 << 0, NVME_CTRL_CTRATT_NON_OP_PSP = 1 << 1, NVME_CTRL_CTRATT_NVM_SETS = 1 << 2, NVME_CTRL_CTRATT_READ_RECV_LVLS = 1 << 3, NVME_CTRL_CTRATT_ENDURANCE_GROUPS = 1 << 4, NVME_CTRL_CTRATT_PREDICTABLE_LAT = 1 << 5, NVME_CTRL_CTRATT_NAMESPACE_GRANULARITY = 1 << 7, NVME_CTRL_CTRATT_UUID_LIST = 1 << 9, NVME_CTRL_SGLS_BYTE_ALIGNED = 1, NVME_CTRL_SGLS_DWORD_ALIGNED = 2, NVME_CTRL_SGLS_KSDBDS = 1 << 2, NVME_CTRL_SGLS_MSDS = 1 << 19, NVME_CTRL_SGLS_SAOS = 1 << 20, }; struct nvme_lbaf { __le16 ms; __u8 ds; __u8 rp; }; struct nvme_id_ns { __le64 nsze; __le64 ncap; __le64 nuse; __u8 nsfeat; __u8 nlbaf; __u8 flbas; __u8 mc; __u8 dpc; __u8 dps; __u8 nmic; __u8 rescap; __u8 fpi; __u8 dlfeat; __le16 nawun; __le16 nawupf; __le16 nacwu; __le16 nabsn; __le16 nabo; __le16 nabspf; __le16 noiob; __u8 nvmcap[16]; __le16 npwg; __le16 npwa; __le16 npdg; __le16 npda; __le16 nows; __u8 rsvd74[18]; __le32 anagrpid; __u8 rsvd96[3]; __u8 nsattr; __le16 nvmsetid; __le16 endgid; __u8 nguid[16]; __u8 eui64[8]; struct nvme_lbaf lbaf[64]; __u8 vs[3712]; }; /* I/O Command Set Independent Identify Namespace Data Structure */ struct nvme_id_ns_cs_indep { __u8 nsfeat; __u8 nmic; __u8 rescap; __u8 fpi; __le32 anagrpid; __u8 nsattr; __u8 rsvd9; __le16 nvmsetid; __le16 endgid; __u8 nstat; __u8 rsvd15[4081]; }; struct nvme_zns_lbafe { __le64 zsze; __u8 zdes; __u8 rsvd9[7]; }; struct nvme_id_ns_zns { __le16 zoc; __le16 ozcs; __le32 mar; __le32 mor; __le32 rrl; __le32 frl; __u8 rsvd20[2796]; struct nvme_zns_lbafe lbafe[64]; __u8 vs[256]; }; struct nvme_id_ctrl_zns { __u8 zasl; __u8 rsvd1[4095]; }; struct nvme_id_ns_nvm { __le64 lbstm; __u8 pic; __u8 rsvd9[3]; __le32 elbaf[64]; __le32 npdgl; __le32 nprg; __le32 npra; __le32 nors; __le32 npdal; __u8 rsvd288[3808]; }; static_assert(sizeof(struct nvme_id_ns_nvm) == 4096); enum { NVME_ID_NS_NVM_STS_MASK = 0x7f, NVME_ID_NS_NVM_GUARD_SHIFT = 7, NVME_ID_NS_NVM_GUARD_MASK = 0x3, NVME_ID_NS_NVM_QPIF_SHIFT = 9, NVME_ID_NS_NVM_QPIF_MASK = 0xf, NVME_ID_NS_NVM_QPIFS = 1 << 3, }; static inline __u8 nvme_elbaf_sts(__u32 elbaf) { return elbaf & NVME_ID_NS_NVM_STS_MASK; } static inline __u8 nvme_elbaf_guard_type(__u32 elbaf) { return (elbaf >> NVME_ID_NS_NVM_GUARD_SHIFT) & NVME_ID_NS_NVM_GUARD_MASK; } static inline __u8 nvme_elbaf_qualified_guard_type(__u32 elbaf) { return (elbaf >> NVME_ID_NS_NVM_QPIF_SHIFT) & NVME_ID_NS_NVM_QPIF_MASK; } struct nvme_id_ctrl_nvm { __u8 vsl; __u8 wzsl; __u8 wusl; __u8 dmrl; __le32 dmrsl; __le64 dmsl; __u8 rsvd16[4080]; }; enum { NVME_ID_CNS_NS = 0x00, NVME_ID_CNS_CTRL = 0x01, NVME_ID_CNS_NS_ACTIVE_LIST = 0x02, NVME_ID_CNS_NS_DESC_LIST = 0x03, NVME_ID_CNS_CS_NS = 0x05, NVME_ID_CNS_CS_CTRL = 0x06, NVME_ID_CNS_NS_ACTIVE_LIST_CS = 0x07, NVME_ID_CNS_NS_CS_INDEP = 0x08, NVME_ID_CNS_NS_PRESENT_LIST = 0x10, NVME_ID_CNS_NS_PRESENT = 0x11, NVME_ID_CNS_CTRL_NS_LIST = 0x12, NVME_ID_CNS_CTRL_LIST = 0x13, NVME_ID_CNS_SCNDRY_CTRL_LIST = 0x15, NVME_ID_CNS_NS_GRANULARITY = 0x16, NVME_ID_CNS_UUID_LIST = 0x17, NVME_ID_CNS_ENDGRP_LIST = 0x19, }; enum { NVME_CSI_NVM = 0, NVME_CSI_ZNS = 2, }; enum { NVME_DIR_IDENTIFY = 0x00, NVME_DIR_STREAMS = 0x01, NVME_DIR_SND_ID_OP_ENABLE = 0x01, NVME_DIR_SND_ST_OP_REL_ID = 0x01, NVME_DIR_SND_ST_OP_REL_RSC = 0x02, NVME_DIR_RCV_ID_OP_PARAM = 0x01, NVME_DIR_RCV_ST_OP_PARAM = 0x01, NVME_DIR_RCV_ST_OP_STATUS = 0x02, NVME_DIR_RCV_ST_OP_RESOURCE = 0x03, NVME_DIR_ENDIR = 0x01, }; enum { NVME_NS_FEAT_THIN = 1 << 0, NVME_NS_FEAT_ATOMICS = 1 << 1, NVME_NS_FEAT_OPTPERF_SHIFT = 4, /* In NVMe version 2.0 and below, OPTPERF is only bit 4 of NSFEAT */ NVME_NS_FEAT_OPTPERF_MASK = 0x1, /* Since version 2.1, OPTPERF is bits 4 and 5 of NSFEAT */ NVME_NS_FEAT_OPTPERF_MASK_2_1 = 0x3, NVME_NS_ATTR_RO = 1 << 0, NVME_NS_FLBAS_LBA_MASK = 0xf, NVME_NS_FLBAS_LBA_UMASK = 0x60, NVME_NS_FLBAS_LBA_SHIFT = 1, NVME_NS_FLBAS_META_EXT = 0x10, NVME_NS_NMIC_SHARED = 1 << 0, NVME_NS_ROTATIONAL = 1 << 4, NVME_NS_VWC_NOT_PRESENT = 1 << 5, NVME_LBAF_RP_BEST = 0, NVME_LBAF_RP_BETTER = 1, NVME_LBAF_RP_GOOD = 2, NVME_LBAF_RP_DEGRADED = 3, NVME_NS_DPC_PI_LAST = 1 << 4, NVME_NS_DPC_PI_FIRST = 1 << 3, NVME_NS_DPC_PI_TYPE3 = 1 << 2, NVME_NS_DPC_PI_TYPE2 = 1 << 1, NVME_NS_DPC_PI_TYPE1 = 1 << 0, NVME_NS_DPS_PI_FIRST = 1 << 3, NVME_NS_DPS_PI_MASK = 0x7, NVME_NS_DPS_PI_TYPE1 = 1, NVME_NS_DPS_PI_TYPE2 = 2, NVME_NS_DPS_PI_TYPE3 = 3, }; enum { NVME_NSTAT_NRDY = 1 << 0, }; enum { NVME_NVM_NS_16B_GUARD = 0, NVME_NVM_NS_32B_GUARD = 1, NVME_NVM_NS_64B_GUARD = 2, NVME_NVM_NS_QTYPE_GUARD = 3, }; static inline __u8 nvme_lbaf_index(__u8 flbas) { return (flbas & NVME_NS_FLBAS_LBA_MASK) | ((flbas & NVME_NS_FLBAS_LBA_UMASK) >> NVME_NS_FLBAS_LBA_SHIFT); } /* Identify Namespace Metadata Capabilities (MC): */ enum { NVME_MC_EXTENDED_LBA = (1 << 0), NVME_MC_METADATA_PTR = (1 << 1), }; struct nvme_ns_id_desc { __u8 nidt; __u8 nidl; __le16 reserved; }; #define NVME_NIDT_EUI64_LEN 8 #define NVME_NIDT_NGUID_LEN 16 #define NVME_NIDT_UUID_LEN 16 #define NVME_NIDT_CSI_LEN 1 enum { NVME_NIDT_EUI64 = 0x01, NVME_NIDT_NGUID = 0x02, NVME_NIDT_UUID = 0x03, NVME_NIDT_CSI = 0x04, }; struct nvme_endurance_group_log { __u8 egcw; __u8 egfeat; __u8 rsvd2; __u8 avsp; __u8 avspt; __u8 pused; __le16 did; __u8 rsvd8[24]; __u8 ee[16]; __u8 dur[16]; __u8 duw[16]; __u8 muw[16]; __u8 hrc[16]; __u8 hwc[16]; __u8 mdie[16]; __u8 neile[16]; __u8 tegcap[16]; __u8 uegcap[16]; __u8 rsvd192[320]; }; struct nvme_rotational_media_log { __le16 endgid; __le16 numa; __le16 nrs; __u8 rsvd6[2]; __le32 spinc; __le32 fspinc; __le32 ldc; __le32 fldc; __u8 rsvd24[488]; }; struct nvme_fdp_config { __u8 flags; #define FDPCFG_FDPE (1U << 0) __u8 fdpcidx; __le16 reserved; }; struct nvme_fdp_ruh_desc { __u8 ruht; __u8 reserved[3]; }; struct nvme_fdp_config_desc { __le16 dsze; __u8 fdpa; __u8 vss; __le32 nrg; __le16 nruh; __le16 maxpids; __le32 nns; __le64 runs; __le32 erutl; __u8 rsvd28[36]; struct nvme_fdp_ruh_desc ruhs[]; }; struct nvme_fdp_config_log { __le16 numfdpc; __u8 ver; __u8 rsvd3; __le32 sze; __u8 rsvd8[8]; /* * This is followed by variable number of nvme_fdp_config_desc * structures, but sparse doesn't like nested variable sized arrays. */ }; struct nvme_smart_log { __u8 critical_warning; __u8 temperature[2]; __u8 avail_spare; __u8 spare_thresh; __u8 percent_used; __u8 endu_grp_crit_warn_sumry; __u8 rsvd7[25]; __u8 data_units_read[16]; __u8 data_units_written[16]; __u8 host_reads[16]; __u8 host_writes[16]; __u8 ctrl_busy_time[16]; __u8 power_cycles[16]; __u8 power_on_hours[16]; __u8 unsafe_shutdowns[16]; __u8 media_errors[16]; __u8 num_err_log_entries[16]; __le32 warning_temp_time; __le32 critical_comp_time; __le16 temp_sensor[8]; __le32 thm_temp1_trans_count; __le32 thm_temp2_trans_count; __le32 thm_temp1_total_time; __le32 thm_temp2_total_time; __u8 rsvd232[280]; }; struct nvme_fw_slot_info_log { __u8 afi; __u8 rsvd1[7]; __le64 frs[7]; __u8 rsvd64[448]; }; enum { NVME_CMD_EFFECTS_CSUPP = 1 << 0, NVME_CMD_EFFECTS_LBCC = 1 << 1, NVME_CMD_EFFECTS_NCC = 1 << 2, NVME_CMD_EFFECTS_NIC = 1 << 3, NVME_CMD_EFFECTS_CCC = 1 << 4, NVME_CMD_EFFECTS_CSER_MASK = GENMASK(15, 14), NVME_CMD_EFFECTS_CSE_MASK = GENMASK(18, 16), NVME_CMD_EFFECTS_UUID_SEL = 1 << 19, NVME_CMD_EFFECTS_SCOPE_MASK = GENMASK(31, 20), }; struct nvme_effects_log { __le32 acs[256]; __le32 iocs[256]; __u8 resv[2048]; }; enum nvme_ana_state { NVME_ANA_OPTIMIZED = 0x01, NVME_ANA_NONOPTIMIZED = 0x02, NVME_ANA_INACCESSIBLE = 0x03, NVME_ANA_PERSISTENT_LOSS = 0x04, NVME_ANA_CHANGE = 0x0f, }; struct nvme_ana_group_desc { __le32 grpid; __le32 nnsids; __le64 chgcnt; __u8 state; __u8 rsvd17[15]; __le32 nsids[]; }; /* flag for the log specific field of the ANA log */ #define NVME_ANA_LOG_RGO (1 << 0) struct nvme_ana_rsp_hdr { __le64 chgcnt; __le16 ngrps; __le16 rsvd10[3]; }; struct nvme_zone_descriptor { __u8 zt; __u8 zs; __u8 za; __u8 rsvd3[5]; __le64 zcap; __le64 zslba; __le64 wp; __u8 rsvd32[32]; }; enum { NVME_ZONE_TYPE_SEQWRITE_REQ = 0x2, }; struct nvme_zone_report { __le64 nr_zones; __u8 resv8[56]; struct nvme_zone_descriptor entries[]; }; enum { NVME_SMART_CRIT_SPARE = 1 << 0, NVME_SMART_CRIT_TEMPERATURE = 1 << 1, NVME_SMART_CRIT_RELIABILITY = 1 << 2, NVME_SMART_CRIT_MEDIA = 1 << 3, NVME_SMART_CRIT_VOLATILE_MEMORY = 1 << 4, }; enum { NVME_AER_ERROR = 0, NVME_AER_SMART = 1, NVME_AER_NOTICE = 2, NVME_AER_CSS = 6, NVME_AER_VS = 7, }; enum { NVME_AER_ERROR_PERSIST_INT_ERR = 0x03, }; enum { NVME_AER_NOTICE_NS_CHANGED = 0x00, NVME_AER_NOTICE_FW_ACT_STARTING = 0x01, NVME_AER_NOTICE_ANA = 0x03, NVME_AER_NOTICE_DISC_CHANGED = 0xf0, }; enum { NVME_AEN_BIT_NS_ATTR = 8, NVME_AEN_BIT_FW_ACT = 9, NVME_AEN_BIT_ANA_CHANGE = 11, NVME_AEN_BIT_DISC_CHANGE = 31, }; enum { NVME_AEN_CFG_NS_ATTR = 1 << NVME_AEN_BIT_NS_ATTR, NVME_AEN_CFG_FW_ACT = 1 << NVME_AEN_BIT_FW_ACT, NVME_AEN_CFG_ANA_CHANGE = 1 << NVME_AEN_BIT_ANA_CHANGE, NVME_AEN_CFG_DISC_CHANGE = 1 << NVME_AEN_BIT_DISC_CHANGE, }; struct nvme_lba_range_type { __u8 type; __u8 attributes; __u8 rsvd2[14]; __le64 slba; __le64 nlb; __u8 guid[16]; __u8 rsvd48[16]; }; enum { NVME_LBART_TYPE_FS = 0x01, NVME_LBART_TYPE_RAID = 0x02, NVME_LBART_TYPE_CACHE = 0x03, NVME_LBART_TYPE_SWAP = 0x04, NVME_LBART_ATTRIB_TEMP = 1 << 0, NVME_LBART_ATTRIB_HIDE = 1 << 1, }; enum nvme_pr_type { NVME_PR_WRITE_EXCLUSIVE = 1, NVME_PR_EXCLUSIVE_ACCESS = 2, NVME_PR_WRITE_EXCLUSIVE_REG_ONLY = 3, NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY = 4, NVME_PR_WRITE_EXCLUSIVE_ALL_REGS = 5, NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS = 6, }; enum nvme_eds { NVME_EXTENDED_DATA_STRUCT = 0x1, }; struct nvme_registered_ctrl { __le16 cntlid; __u8 rcsts; __u8 rsvd3[5]; __le64 hostid; __le64 rkey; }; struct nvme_reservation_status { __le32 gen; __u8 rtype; __u8 regctl[2]; __u8 resv5[2]; __u8 ptpls; __u8 resv10[14]; struct nvme_registered_ctrl regctl_ds[]; }; struct nvme_registered_ctrl_ext { __le16 cntlid; __u8 rcsts; __u8 rsvd3[5]; __le64 rkey; __u8 hostid[16]; __u8 rsvd32[32]; }; struct nvme_reservation_status_ext { __le32 gen; __u8 rtype; __u8 regctl[2]; __u8 resv5[2]; __u8 ptpls; __u8 resv10[14]; __u8 rsvd24[40]; struct nvme_registered_ctrl_ext regctl_eds[]; }; /* I/O commands */ enum nvme_opcode { nvme_cmd_flush = 0x00, nvme_cmd_write = 0x01, nvme_cmd_read = 0x02, nvme_cmd_write_uncor = 0x04, nvme_cmd_compare = 0x05, nvme_cmd_write_zeroes = 0x08, nvme_cmd_dsm = 0x09, nvme_cmd_verify = 0x0c, nvme_cmd_resv_register = 0x0d, nvme_cmd_resv_report = 0x0e, nvme_cmd_resv_acquire = 0x11, nvme_cmd_io_mgmt_recv = 0x12, nvme_cmd_resv_release = 0x15, nvme_cmd_zone_mgmt_send = 0x79, nvme_cmd_zone_mgmt_recv = 0x7a, nvme_cmd_zone_append = 0x7d, nvme_cmd_vendor_start = 0x80, }; #define nvme_opcode_name(opcode) { opcode, #opcode } #define show_nvm_opcode_name(val) \ __print_symbolic(val, \ nvme_opcode_name(nvme_cmd_flush), \ nvme_opcode_name(nvme_cmd_write), \ nvme_opcode_name(nvme_cmd_read), \ nvme_opcode_name(nvme_cmd_write_uncor), \ nvme_opcode_name(nvme_cmd_compare), \ nvme_opcode_name(nvme_cmd_write_zeroes), \ nvme_opcode_name(nvme_cmd_dsm), \ nvme_opcode_name(nvme_cmd_verify), \ nvme_opcode_name(nvme_cmd_resv_register), \ nvme_opcode_name(nvme_cmd_resv_report), \ nvme_opcode_name(nvme_cmd_resv_acquire), \ nvme_opcode_name(nvme_cmd_io_mgmt_recv), \ nvme_opcode_name(nvme_cmd_resv_release), \ nvme_opcode_name(nvme_cmd_zone_mgmt_send), \ nvme_opcode_name(nvme_cmd_zone_mgmt_recv), \ nvme_opcode_name(nvme_cmd_zone_append)) /* * Descriptor subtype - lower 4 bits of nvme_(keyed_)sgl_desc identifier * * @NVME_SGL_FMT_ADDRESS: absolute address of the data block * @NVME_SGL_FMT_OFFSET: relative offset of the in-capsule data block * @NVME_SGL_FMT_TRANSPORT_A: transport defined format, value 0xA * @NVME_SGL_FMT_INVALIDATE: RDMA transport specific remote invalidation * request subtype */ enum { NVME_SGL_FMT_ADDRESS = 0x00, NVME_SGL_FMT_OFFSET = 0x01, NVME_SGL_FMT_TRANSPORT_A = 0x0A, NVME_SGL_FMT_INVALIDATE = 0x0f, }; /* * Descriptor type - upper 4 bits of nvme_(keyed_)sgl_desc identifier * * For struct nvme_sgl_desc: * @NVME_SGL_FMT_DATA_DESC: data block descriptor * @NVME_SGL_FMT_SEG_DESC: sgl segment descriptor * @NVME_SGL_FMT_LAST_SEG_DESC: last sgl segment descriptor * * For struct nvme_keyed_sgl_desc: * @NVME_KEY_SGL_FMT_DATA_DESC: keyed data block descriptor * * Transport-specific SGL types: * @NVME_TRANSPORT_SGL_DATA_DESC: Transport SGL data dlock descriptor */ enum { NVME_SGL_FMT_DATA_DESC = 0x00, NVME_SGL_FMT_SEG_DESC = 0x02, NVME_SGL_FMT_LAST_SEG_DESC = 0x03, NVME_KEY_SGL_FMT_DATA_DESC = 0x04, NVME_TRANSPORT_SGL_DATA_DESC = 0x05, }; struct nvme_sgl_desc { __le64 addr; __le32 length; __u8 rsvd[3]; __u8 type; }; struct nvme_keyed_sgl_desc { __le64 addr; __u8 length[3]; __u8 key[4]; __u8 type; }; union nvme_data_ptr { struct { __le64 prp1; __le64 prp2; }; struct nvme_sgl_desc sgl; struct nvme_keyed_sgl_desc ksgl; }; /* * Lowest two bits of our flags field (FUSE field in the spec): * * @NVME_CMD_FUSE_FIRST: Fused Operation, first command * @NVME_CMD_FUSE_SECOND: Fused Operation, second command * * Highest two bits in our flags field (PSDT field in the spec): * * @NVME_CMD_PSDT_SGL_METABUF: Use SGLS for this transfer, * If used, MPTR contains addr of single physical buffer (byte aligned). * @NVME_CMD_PSDT_SGL_METASEG: Use SGLS for this transfer, * If used, MPTR contains an address of an SGL segment containing * exactly 1 SGL descriptor (qword aligned). */ enum { NVME_CMD_FUSE_FIRST = (1 << 0), NVME_CMD_FUSE_SECOND = (1 << 1), NVME_CMD_SGL_METABUF = (1 << 6), NVME_CMD_SGL_METASEG = (1 << 7), NVME_CMD_SGL_ALL = NVME_CMD_SGL_METABUF | NVME_CMD_SGL_METASEG, }; struct nvme_common_command { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __le32 cdw2[2]; __le64 metadata; union nvme_data_ptr dptr; struct_group(cdws, __le32 cdw10; __le32 cdw11; __le32 cdw12; __le32 cdw13; __le32 cdw14; __le32 cdw15; ); }; struct nvme_rw_command { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __le32 cdw2; __le32 cdw3; __le64 metadata; union nvme_data_ptr dptr; __le64 slba; __le16 length; __le16 control; __le32 dsmgmt; __le32 reftag; __le16 lbat; __le16 lbatm; }; enum { NVME_RW_LR = 1 << 15, NVME_RW_FUA = 1 << 14, NVME_RW_APPEND_PIREMAP = 1 << 9, NVME_RW_DSM_FREQ_UNSPEC = 0, NVME_RW_DSM_FREQ_TYPICAL = 1, NVME_RW_DSM_FREQ_RARE = 2, NVME_RW_DSM_FREQ_READS = 3, NVME_RW_DSM_FREQ_WRITES = 4, NVME_RW_DSM_FREQ_RW = 5, NVME_RW_DSM_FREQ_ONCE = 6, NVME_RW_DSM_FREQ_PREFETCH = 7, NVME_RW_DSM_FREQ_TEMP = 8, NVME_RW_DSM_LATENCY_NONE = 0 << 4, NVME_RW_DSM_LATENCY_IDLE = 1 << 4, NVME_RW_DSM_LATENCY_NORM = 2 << 4, NVME_RW_DSM_LATENCY_LOW = 3 << 4, NVME_RW_DSM_SEQ_REQ = 1 << 6, NVME_RW_DSM_COMPRESSED = 1 << 7, NVME_RW_PRINFO_PRCHK_REF = 1 << 10, NVME_RW_PRINFO_PRCHK_APP = 1 << 11, NVME_RW_PRINFO_PRCHK_GUARD = 1 << 12, NVME_RW_PRINFO_PRACT = 1 << 13, NVME_RW_DTYPE_STREAMS = 1 << 4, NVME_RW_DTYPE_DPLCMT = 2 << 4, NVME_WZ_DEAC = 1 << 9, }; struct nvme_dsm_cmd { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __u64 rsvd2[2]; union nvme_data_ptr dptr; __le32 nr; __le32 attributes; __u32 rsvd12[4]; }; enum { NVME_DSMGMT_IDR = 1 << 0, NVME_DSMGMT_IDW = 1 << 1, NVME_DSMGMT_AD = 1 << 2, }; #define NVME_DSM_MAX_RANGES 256 struct nvme_dsm_range { __le32 cattr; __le32 nlb; __le64 slba; }; struct nvme_write_zeroes_cmd { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __u64 rsvd2; __le64 metadata; union nvme_data_ptr dptr; __le64 slba; __le16 length; __le16 control; __le32 dsmgmt; __le32 reftag; __le16 lbat; __le16 lbatm; }; enum nvme_zone_mgmt_action { NVME_ZONE_CLOSE = 0x1, NVME_ZONE_FINISH = 0x2, NVME_ZONE_OPEN = 0x3, NVME_ZONE_RESET = 0x4, NVME_ZONE_OFFLINE = 0x5, NVME_ZONE_SET_DESC_EXT = 0x10, }; struct nvme_zone_mgmt_send_cmd { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __le32 cdw2[2]; __le64 metadata; union nvme_data_ptr dptr; __le64 slba; __le32 cdw12; __u8 zsa; __u8 select_all; __u8 rsvd13[2]; __le32 cdw14[2]; }; struct nvme_zone_mgmt_recv_cmd { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __le64 rsvd2[2]; union nvme_data_ptr dptr; __le64 slba; __le32 numd; __u8 zra; __u8 zrasf; __u8 pr; __u8 rsvd13; __le32 cdw14[2]; }; struct nvme_io_mgmt_recv_cmd { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __le64 rsvd2[2]; union nvme_data_ptr dptr; __u8 mo; __u8 rsvd11; __u16 mos; __le32 numd; __le32 cdw12[4]; }; enum { NVME_IO_MGMT_RECV_MO_RUHS = 1, }; struct nvme_fdp_ruh_status_desc { __le16 pid; __le16 ruhid; __le32 earutr; __le64 ruamw; __u8 reserved[16]; }; struct nvme_fdp_ruh_status { __u8 rsvd0[14]; __le16 nruhsd; struct nvme_fdp_ruh_status_desc ruhsd[]; }; enum { NVME_ZRA_ZONE_REPORT = 0, NVME_ZRASF_ZONE_REPORT_ALL = 0, NVME_ZRASF_ZONE_STATE_EMPTY = 0x01, NVME_ZRASF_ZONE_STATE_IMP_OPEN = 0x02, NVME_ZRASF_ZONE_STATE_EXP_OPEN = 0x03, NVME_ZRASF_ZONE_STATE_CLOSED = 0x04, NVME_ZRASF_ZONE_STATE_READONLY = 0x05, NVME_ZRASF_ZONE_STATE_FULL = 0x06, NVME_ZRASF_ZONE_STATE_OFFLINE = 0x07, NVME_REPORT_ZONE_PARTIAL = 1, }; /* Features */ enum { NVME_TEMP_THRESH_MASK = 0xffff, NVME_TEMP_THRESH_SELECT_SHIFT = 16, NVME_TEMP_THRESH_TYPE_UNDER = 0x100000, }; struct nvme_feat_auto_pst { __le64 entries[32]; }; enum { NVME_HOST_MEM_ENABLE = (1 << 0), NVME_HOST_MEM_RETURN = (1 << 1), }; struct nvme_feat_host_behavior { __u8 acre; __u8 etdas; __u8 lbafee; __u8 resv1[509]; }; enum { NVME_ENABLE_ACRE = 1, NVME_ENABLE_LBAFEE = 1, }; /* Admin commands */ enum nvme_admin_opcode { nvme_admin_delete_sq = 0x00, nvme_admin_create_sq = 0x01, nvme_admin_get_log_page = 0x02, nvme_admin_delete_cq = 0x04, nvme_admin_create_cq = 0x05, nvme_admin_identify = 0x06, nvme_admin_abort_cmd = 0x08, nvme_admin_set_features = 0x09, nvme_admin_get_features = 0x0a, nvme_admin_async_event = 0x0c, nvme_admin_ns_mgmt = 0x0d, nvme_admin_activate_fw = 0x10, nvme_admin_download_fw = 0x11, nvme_admin_dev_self_test = 0x14, nvme_admin_ns_attach = 0x15, nvme_admin_keep_alive = 0x18, nvme_admin_directive_send = 0x19, nvme_admin_directive_recv = 0x1a, nvme_admin_virtual_mgmt = 0x1c, nvme_admin_nvme_mi_send = 0x1d, nvme_admin_nvme_mi_recv = 0x1e, nvme_admin_dbbuf = 0x7C, nvme_admin_format_nvm = 0x80, nvme_admin_security_send = 0x81, nvme_admin_security_recv = 0x82, nvme_admin_sanitize_nvm = 0x84, nvme_admin_get_lba_status = 0x86, nvme_admin_vendor_start = 0xC0, }; #define nvme_admin_opcode_name(opcode) { opcode, #opcode } #define show_admin_opcode_name(val) \ __print_symbolic(val, \ nvme_admin_opcode_name(nvme_admin_delete_sq), \ nvme_admin_opcode_name(nvme_admin_create_sq), \ nvme_admin_opcode_name(nvme_admin_get_log_page), \ nvme_admin_opcode_name(nvme_admin_delete_cq), \ nvme_admin_opcode_name(nvme_admin_create_cq), \ nvme_admin_opcode_name(nvme_admin_identify), \ nvme_admin_opcode_name(nvme_admin_abort_cmd), \ nvme_admin_opcode_name(nvme_admin_set_features), \ nvme_admin_opcode_name(nvme_admin_get_features), \ nvme_admin_opcode_name(nvme_admin_async_event), \ nvme_admin_opcode_name(nvme_admin_ns_mgmt), \ nvme_admin_opcode_name(nvme_admin_activate_fw), \ nvme_admin_opcode_name(nvme_admin_download_fw), \ nvme_admin_opcode_name(nvme_admin_dev_self_test), \ nvme_admin_opcode_name(nvme_admin_ns_attach), \ nvme_admin_opcode_name(nvme_admin_keep_alive), \ nvme_admin_opcode_name(nvme_admin_directive_send), \ nvme_admin_opcode_name(nvme_admin_directive_recv), \ nvme_admin_opcode_name(nvme_admin_virtual_mgmt), \ nvme_admin_opcode_name(nvme_admin_nvme_mi_send), \ nvme_admin_opcode_name(nvme_admin_nvme_mi_recv), \ nvme_admin_opcode_name(nvme_admin_dbbuf), \ nvme_admin_opcode_name(nvme_admin_format_nvm), \ nvme_admin_opcode_name(nvme_admin_security_send), \ nvme_admin_opcode_name(nvme_admin_security_recv), \ nvme_admin_opcode_name(nvme_admin_sanitize_nvm), \ nvme_admin_opcode_name(nvme_admin_get_lba_status)) enum { NVME_QUEUE_PHYS_CONTIG = (1 << 0), NVME_CQ_IRQ_ENABLED = (1 << 1), NVME_SQ_PRIO_URGENT = (0 << 1), NVME_SQ_PRIO_HIGH = (1 << 1), NVME_SQ_PRIO_MEDIUM = (2 << 1), NVME_SQ_PRIO_LOW = (3 << 1), NVME_FEAT_ARBITRATION = 0x01, NVME_FEAT_POWER_MGMT = 0x02, NVME_FEAT_LBA_RANGE = 0x03, NVME_FEAT_TEMP_THRESH = 0x04, NVME_FEAT_ERR_RECOVERY = 0x05, NVME_FEAT_VOLATILE_WC = 0x06, NVME_FEAT_NUM_QUEUES = 0x07, NVME_FEAT_IRQ_COALESCE = 0x08, NVME_FEAT_IRQ_CONFIG = 0x09, NVME_FEAT_WRITE_ATOMIC = 0x0a, NVME_FEAT_ASYNC_EVENT = 0x0b, NVME_FEAT_AUTO_PST = 0x0c, NVME_FEAT_HOST_MEM_BUF = 0x0d, NVME_FEAT_TIMESTAMP = 0x0e, NVME_FEAT_KATO = 0x0f, NVME_FEAT_HCTM = 0x10, NVME_FEAT_NOPSC = 0x11, NVME_FEAT_RRL = 0x12, NVME_FEAT_PLM_CONFIG = 0x13, NVME_FEAT_PLM_WINDOW = 0x14, NVME_FEAT_HOST_BEHAVIOR = 0x16, NVME_FEAT_SANITIZE = 0x17, NVME_FEAT_FDP = 0x1d, NVME_FEAT_SW_PROGRESS = 0x80, NVME_FEAT_HOST_ID = 0x81, NVME_FEAT_RESV_MASK = 0x82, NVME_FEAT_RESV_PERSIST = 0x83, NVME_FEAT_WRITE_PROTECT = 0x84, NVME_FEAT_VENDOR_START = 0xC0, NVME_FEAT_VENDOR_END = 0xFF, NVME_LOG_SUPPORTED = 0x00, NVME_LOG_ERROR = 0x01, NVME_LOG_SMART = 0x02, NVME_LOG_FW_SLOT = 0x03, NVME_LOG_CHANGED_NS = 0x04, NVME_LOG_CMD_EFFECTS = 0x05, NVME_LOG_DEVICE_SELF_TEST = 0x06, NVME_LOG_TELEMETRY_HOST = 0x07, NVME_LOG_TELEMETRY_CTRL = 0x08, NVME_LOG_ENDURANCE_GROUP = 0x09, NVME_LOG_ANA = 0x0c, NVME_LOG_FEATURES = 0x12, NVME_LOG_RMI = 0x16, NVME_LOG_FDP_CONFIGS = 0x20, NVME_LOG_DISC = 0x70, NVME_LOG_RESERVATION = 0x80, NVME_FWACT_REPL = (0 << 3), NVME_FWACT_REPL_ACTV = (1 << 3), NVME_FWACT_ACTV = (2 << 3), }; struct nvme_supported_log { __le32 lids[256]; }; enum { NVME_LIDS_LSUPP = 1 << 0, }; struct nvme_supported_features_log { __le32 fis[256]; }; enum { NVME_FIS_FSUPP = 1 << 0, NVME_FIS_NSCPE = 1 << 20, NVME_FIS_CSCPE = 1 << 21, }; /* NVMe Namespace Write Protect State */ enum { NVME_NS_NO_WRITE_PROTECT = 0, NVME_NS_WRITE_PROTECT, NVME_NS_WRITE_PROTECT_POWER_CYCLE, NVME_NS_WRITE_PROTECT_PERMANENT, }; #define NVME_MAX_CHANGED_NAMESPACES 1024 struct nvme_identify { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __u64 rsvd2[2]; union nvme_data_ptr dptr; __u8 cns; __u8 rsvd3; __le16 ctrlid; __le16 cnssid; __u8 rsvd11; __u8 csi; __u32 rsvd12[4]; }; #define NVME_IDENTIFY_DATA_SIZE 4096 struct nvme_features { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __u64 rsvd2[2]; union nvme_data_ptr dptr; __le32 fid; __le32 dword11; __le32 dword12; __le32 dword13; __le32 dword14; __le32 dword15; }; struct nvme_host_mem_buf_desc { __le64 addr; __le32 size; __u32 rsvd; }; struct nvme_create_cq { __u8 opcode; __u8 flags; __u16 command_id; __u32 rsvd1[5]; __le64 prp1; __u64 rsvd8; __le16 cqid; __le16 qsize; __le16 cq_flags; __le16 irq_vector; __u32 rsvd12[4]; }; struct nvme_create_sq { __u8 opcode; __u8 flags; __u16 command_id; __u32 rsvd1[5]; __le64 prp1; __u64 rsvd8; __le16 sqid; __le16 qsize; __le16 sq_flags; __le16 cqid; __u32 rsvd12[4]; }; struct nvme_delete_queue { __u8 opcode; __u8 flags; __u16 command_id; __u32 rsvd1[9]; __le16 qid; __u16 rsvd10; __u32 rsvd11[5]; }; struct nvme_abort_cmd { __u8 opcode; __u8 flags; __u16 command_id; __u32 rsvd1[9]; __le16 sqid; __u16 cid; __u32 rsvd11[5]; }; struct nvme_download_firmware { __u8 opcode; __u8 flags; __u16 command_id; __u32 rsvd1[5]; union nvme_data_ptr dptr; __le32 numd; __le32 offset; __u32 rsvd12[4]; }; struct nvme_format_cmd { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __u64 rsvd2[4]; __le32 cdw10; __u32 rsvd11[5]; }; struct nvme_get_log_page_command { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __u64 rsvd2[2]; union nvme_data_ptr dptr; __u8 lid; __u8 lsp; /* upper 4 bits reserved */ __le16 numdl; __le16 numdu; __le16 lsi; union { struct { __le32 lpol; __le32 lpou; }; __le64 lpo; }; __u8 rsvd14[3]; __u8 csi; __u32 rsvd15; }; struct nvme_directive_cmd { __u8 opcode; __u8 flags; __u16 command_id; __le32 nsid; __u64 rsvd2[2]; union nvme_data_ptr dptr; __le32 numd; __u8 doper; __u8 dtype; __le16 dspec; __u8 endir; __u8 tdtype; __u16 rsvd15; __u32 rsvd16[3]; }; /* * Fabrics subcommands. */ enum nvmf_fabrics_opcode { nvme_fabrics_command = 0x7f, }; enum nvmf_capsule_command { nvme_fabrics_type_property_set = 0x00, nvme_fabrics_type_connect = 0x01, nvme_fabrics_type_property_get = 0x04, nvme_fabrics_type_auth_send = 0x05, nvme_fabrics_type_auth_receive = 0x06, }; #define nvme_fabrics_type_name(type) { type, #type } #define show_fabrics_type_name(type) \ __print_symbolic(type, \ nvme_fabrics_type_name(nvme_fabrics_type_property_set), \ nvme_fabrics_type_name(nvme_fabrics_type_connect), \ nvme_fabrics_type_name(nvme_fabrics_type_property_get), \ nvme_fabrics_type_name(nvme_fabrics_type_auth_send), \ nvme_fabrics_type_name(nvme_fabrics_type_auth_receive)) /* * If not fabrics command, fctype will be ignored. */ #define show_opcode_name(qid, opcode, fctype) \ ((opcode) == nvme_fabrics_command ? \ show_fabrics_type_name(fctype) : \ ((qid) ? \ show_nvm_opcode_name(opcode) : \ show_admin_opcode_name(opcode))) struct nvmf_common_command { __u8 opcode; __u8 resv1; __u16 command_id; __u8 fctype; __u8 resv2[35]; __u8 ts[24]; }; /* * The legal cntlid range a NVMe Target will provide. * Note that cntlid of value 0 is considered illegal in the fabrics world. * Devices based on earlier specs did not have the subsystem concept; * therefore, those devices had their cntlid value set to 0 as a result. */ #define NVME_CNTLID_MIN 1 #define NVME_CNTLID_MAX 0xffef #define NVME_CNTLID_DYNAMIC 0xffff #define MAX_DISC_LOGS 255 /* Discovery log page entry flags (EFLAGS): */ enum { NVME_DISC_EFLAGS_EPCSD = (1 << 1), NVME_DISC_EFLAGS_DUPRETINFO = (1 << 0), }; /* Discovery log page entry */ struct nvmf_disc_rsp_page_entry { __u8 trtype; __u8 adrfam; __u8 subtype; __u8 treq; __le16 portid; __le16 cntlid; __le16 asqsz; __le16 eflags; __u8 resv10[20]; char trsvcid[NVMF_TRSVCID_SIZE]; __u8 resv64[192]; char subnqn[NVMF_NQN_FIELD_LEN]; char traddr[NVMF_TRADDR_SIZE]; union tsas { char common[NVMF_TSAS_SIZE]; struct rdma { __u8 qptype; __u8 prtype; __u8 cms; __u8 resv3[5]; __u16 pkey; __u8 resv10[246]; } rdma; struct tcp { __u8 sectype; } tcp; } tsas; }; /* Discovery log page header */ struct nvmf_disc_rsp_page_hdr { __le64 genctr; __le64 numrec; __le16 recfmt; __u8 resv14[1006]; struct nvmf_disc_rsp_page_entry entries[]; }; enum { NVME_CONNECT_DISABLE_SQFLOW = (1 << 2), }; struct nvmf_connect_command { __u8 opcode; __u8 resv1; __u16 command_id; __u8 fctype; __u8 resv2[19]; union nvme_data_ptr dptr; __le16 recfmt; __le16 qid; __le16 sqsize; __u8 cattr; __u8 resv3; __le32 kato; __u8 resv4[12]; }; enum { NVME_CONNECT_AUTHREQ_ASCR = (1U << 18), NVME_CONNECT_AUTHREQ_ATR = (1U << 17), }; struct nvmf_connect_data { uuid_t hostid; __le16 cntlid; char resv4[238]; char subsysnqn[NVMF_NQN_FIELD_LEN]; char hostnqn[NVMF_NQN_FIELD_LEN]; char resv5[256]; }; struct nvmf_property_set_command { __u8 opcode; __u8 resv1; __u16 command_id; __u8 fctype; __u8 resv2[35]; __u8 attrib; __u8 resv3[3]; __le32 offset; __le64 value; __u8 resv4[8]; }; struct nvmf_property_get_command { __u8 opcode; __u8 resv1; __u16 command_id; __u8 fctype; __u8 resv2[35]; __u8 attrib; __u8 resv3[3]; __le32 offset; __u8 resv4[16]; }; struct nvmf_auth_common_command { __u8 opcode; __u8 resv1; __u16 command_id; __u8 fctype; __u8 resv2[19]; union nvme_data_ptr dptr; __u8 resv3; __u8 spsp0; __u8 spsp1; __u8 secp; __le32 al_tl; __u8 resv4[16]; }; struct nvmf_auth_send_command { __u8 opcode; __u8 resv1; __u16 command_id; __u8 fctype; __u8 resv2[19]; union nvme_data_ptr dptr; __u8 resv3; __u8 spsp0; __u8 spsp1; __u8 secp; __le32 tl; __u8 resv4[16]; }; struct nvmf_auth_receive_command { __u8 opcode; __u8 resv1; __u16 command_id; __u8 fctype; __u8 resv2[19]; union nvme_data_ptr dptr; __u8 resv3; __u8 spsp0; __u8 spsp1; __u8 secp; __le32 al; __u8 resv4[16]; }; /* Value for secp */ enum { NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER = 0xe9, }; /* Defined value for auth_type */ enum { NVME_AUTH_COMMON_MESSAGES = 0x00, NVME_AUTH_DHCHAP_MESSAGES = 0x01, }; /* Defined messages for auth_id */ enum { NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE = 0x00, NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE = 0x01, NVME_AUTH_DHCHAP_MESSAGE_REPLY = 0x02, NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1 = 0x03, NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 = 0x04, NVME_AUTH_DHCHAP_MESSAGE_FAILURE2 = 0xf0, NVME_AUTH_DHCHAP_MESSAGE_FAILURE1 = 0xf1, }; struct nvmf_auth_dhchap_protocol_descriptor { __u8 authid; __u8 rsvd; __u8 halen; __u8 dhlen; __u8 idlist[60]; }; enum { NVME_AUTH_DHCHAP_AUTH_ID = 0x01, }; /* Defined hash functions for DH-HMAC-CHAP authentication */ enum { NVME_AUTH_HASH_SHA256 = 0x01, NVME_AUTH_HASH_SHA384 = 0x02, NVME_AUTH_HASH_SHA512 = 0x03, NVME_AUTH_HASH_INVALID = 0xff, }; /* Maximum digest size for any NVME_AUTH_HASH_* value */ enum { NVME_AUTH_MAX_DIGEST_SIZE = 64, }; /* Defined Diffie-Hellman group identifiers for DH-HMAC-CHAP authentication */ enum { NVME_AUTH_DHGROUP_NULL = 0x00, NVME_AUTH_DHGROUP_2048 = 0x01, NVME_AUTH_DHGROUP_3072 = 0x02, NVME_AUTH_DHGROUP_4096 = 0x03, NVME_AUTH_DHGROUP_6144 = 0x04, NVME_AUTH_DHGROUP_8192 = 0x05, NVME_AUTH_DHGROUP_INVALID = 0xff, }; enum { NVME_AUTH_SECP_NOSC = 0x00, NVME_AUTH_SECP_SC = 0x01, NVME_AUTH_SECP_NEWTLSPSK = 0x02, NVME_AUTH_SECP_REPLACETLSPSK = 0x03, }; union nvmf_auth_protocol { struct nvmf_auth_dhchap_protocol_descriptor dhchap; }; struct nvmf_auth_dhchap_negotiate_data { __u8 auth_type; __u8 auth_id; __le16 rsvd; __le16 t_id; __u8 sc_c; __u8 napd; union nvmf_auth_protocol auth_protocol[]; }; struct nvmf_auth_dhchap_challenge_data { __u8 auth_type; __u8 auth_id; __u16 rsvd1; __le16 t_id; __u8 hl; __u8 rsvd2; __u8 hashid; __u8 dhgid; __le16 dhvlen; __le32 seqnum; /* 'hl' bytes of challenge value */ __u8 cval[]; /* followed by 'dhvlen' bytes of DH value */ }; struct nvmf_auth_dhchap_reply_data { __u8 auth_type; __u8 auth_id; __le16 rsvd1; __le16 t_id; __u8 hl; __u8 rsvd2; __u8 cvalid; __u8 rsvd3; __le16 dhvlen; __le32 seqnum; /* 'hl' bytes of response data */ __u8 rval[]; /* followed by 'hl' bytes of Challenge value */ /* followed by 'dhvlen' bytes of DH value */ }; enum { NVME_AUTH_DHCHAP_RESPONSE_VALID = (1 << 0), }; struct nvmf_auth_dhchap_success1_data { __u8 auth_type; __u8 auth_id; __le16 rsvd1; __le16 t_id; __u8 hl; __u8 rsvd2; __u8 rvalid; __u8 rsvd3[7]; /* 'hl' bytes of response value */ __u8 rval[]; }; struct nvmf_auth_dhchap_success2_data { __u8 auth_type; __u8 auth_id; __le16 rsvd1; __le16 t_id; __u8 rsvd2[10]; }; struct nvmf_auth_dhchap_failure_data { __u8 auth_type; __u8 auth_id; __le16 rsvd1; __le16 t_id; __u8 rescode; __u8 rescode_exp; }; enum { NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED = 0x01, }; enum { NVME_AUTH_DHCHAP_FAILURE_FAILED = 0x01, NVME_AUTH_DHCHAP_FAILURE_NOT_USABLE = 0x02, NVME_AUTH_DHCHAP_FAILURE_CONCAT_MISMATCH = 0x03, NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE = 0x04, NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE = 0x05, NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD = 0x06, NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE = 0x07, }; struct nvme_dbbuf { __u8 opcode; __u8 flags; __u16 command_id; __u32 rsvd1[5]; __le64 prp1; __le64 prp2; __u32 rsvd12[6]; }; struct streams_directive_params { __le16 msl; __le16 nssa; __le16 nsso; __u8 rsvd[10]; __le32 sws; __le16 sgs; __le16 nsa; __le16 nso; __u8 rsvd2[6]; }; struct nvme_command { union { struct nvme_common_command common; struct nvme_rw_command rw; struct nvme_identify identify; struct nvme_features features; struct nvme_create_cq create_cq; struct nvme_create_sq create_sq; struct nvme_delete_queue delete_queue; struct nvme_download_firmware dlfw; struct nvme_format_cmd format; struct nvme_dsm_cmd dsm; struct nvme_write_zeroes_cmd write_zeroes; struct nvme_zone_mgmt_send_cmd zms; struct nvme_zone_mgmt_recv_cmd zmr; struct nvme_abort_cmd abort; struct nvme_get_log_page_command get_log_page; struct nvmf_common_command fabrics; struct nvmf_connect_command connect; struct nvmf_property_set_command prop_set; struct nvmf_property_get_command prop_get; struct nvmf_auth_common_command auth_common; struct nvmf_auth_send_command auth_send; struct nvmf_auth_receive_command auth_receive; struct nvme_dbbuf dbbuf; struct nvme_directive_cmd directive; struct nvme_io_mgmt_recv_cmd imr; }; }; static inline bool nvme_is_fabrics(const struct nvme_command *cmd) { return cmd->common.opcode == nvme_fabrics_command; } #ifdef CONFIG_NVME_VERBOSE_ERRORS const char *nvme_get_error_status_str(u16 status); const char *nvme_get_opcode_str(u8 opcode); const char *nvme_get_admin_opcode_str(u8 opcode); const char *nvme_get_fabrics_opcode_str(u8 opcode); #else /* CONFIG_NVME_VERBOSE_ERRORS */ static inline const char *nvme_get_error_status_str(u16 status) { return "I/O Error"; } static inline const char *nvme_get_opcode_str(u8 opcode) { return "I/O Cmd"; } static inline const char *nvme_get_admin_opcode_str(u8 opcode) { return "Admin Cmd"; } static inline const char *nvme_get_fabrics_opcode_str(u8 opcode) { return "Fabrics Cmd"; } #endif /* CONFIG_NVME_VERBOSE_ERRORS */ static inline const char *nvme_opcode_str(int qid, u8 opcode) { return qid ? nvme_get_opcode_str(opcode) : nvme_get_admin_opcode_str(opcode); } static inline const char *nvme_fabrics_opcode_str( int qid, const struct nvme_command *cmd) { if (nvme_is_fabrics(cmd)) return nvme_get_fabrics_opcode_str(cmd->fabrics.fctype); return nvme_opcode_str(qid, cmd->common.opcode); } struct nvme_error_slot { __le64 error_count; __le16 sqid; __le16 cmdid; __le16 status_field; __le16 param_error_location; __le64 lba; __le32 nsid; __u8 vs; __u8 resv[3]; __le64 cs; __u8 resv2[24]; }; static inline bool nvme_is_write(const struct nvme_command *cmd) { /* * What a mess... * * Why can't we simply have a Fabrics In and Fabrics out command? */ if (unlikely(nvme_is_fabrics(cmd))) return cmd->fabrics.fctype & 1; return cmd->common.opcode & 1; } enum { /* * Generic Command Status: */ NVME_SCT_GENERIC = 0x0, NVME_SC_SUCCESS = 0x0, NVME_SC_INVALID_OPCODE = 0x1, NVME_SC_INVALID_FIELD = 0x2, NVME_SC_CMDID_CONFLICT = 0x3, NVME_SC_DATA_XFER_ERROR = 0x4, NVME_SC_POWER_LOSS = 0x5, NVME_SC_INTERNAL = 0x6, NVME_SC_ABORT_REQ = 0x7, NVME_SC_ABORT_QUEUE = 0x8, NVME_SC_FUSED_FAIL = 0x9, NVME_SC_FUSED_MISSING = 0xa, NVME_SC_INVALID_NS = 0xb, NVME_SC_CMD_SEQ_ERROR = 0xc, NVME_SC_SGL_INVALID_LAST = 0xd, NVME_SC_SGL_INVALID_COUNT = 0xe, NVME_SC_SGL_INVALID_DATA = 0xf, NVME_SC_SGL_INVALID_METADATA = 0x10, NVME_SC_SGL_INVALID_TYPE = 0x11, NVME_SC_CMB_INVALID_USE = 0x12, NVME_SC_PRP_INVALID_OFFSET = 0x13, NVME_SC_ATOMIC_WU_EXCEEDED = 0x14, NVME_SC_OP_DENIED = 0x15, NVME_SC_SGL_INVALID_OFFSET = 0x16, NVME_SC_RESERVED = 0x17, NVME_SC_HOST_ID_INCONSIST = 0x18, NVME_SC_KA_TIMEOUT_EXPIRED = 0x19, NVME_SC_KA_TIMEOUT_INVALID = 0x1A, NVME_SC_ABORTED_PREEMPT_ABORT = 0x1B, NVME_SC_SANITIZE_FAILED = 0x1C, NVME_SC_SANITIZE_IN_PROGRESS = 0x1D, NVME_SC_SGL_INVALID_GRANULARITY = 0x1E, NVME_SC_CMD_NOT_SUP_CMB_QUEUE = 0x1F, NVME_SC_NS_WRITE_PROTECTED = 0x20, NVME_SC_CMD_INTERRUPTED = 0x21, NVME_SC_TRANSIENT_TR_ERR = 0x22, NVME_SC_ADMIN_COMMAND_MEDIA_NOT_READY = 0x24, NVME_SC_INVALID_IO_CMD_SET = 0x2C, NVME_SC_LBA_RANGE = 0x80, NVME_SC_CAP_EXCEEDED = 0x81, NVME_SC_NS_NOT_READY = 0x82, NVME_SC_RESERVATION_CONFLICT = 0x83, NVME_SC_FORMAT_IN_PROGRESS = 0x84, /* * Command Specific Status: */ NVME_SCT_COMMAND_SPECIFIC = 0x100, NVME_SC_CQ_INVALID = 0x100, NVME_SC_QID_INVALID = 0x101, NVME_SC_QUEUE_SIZE = 0x102, NVME_SC_ABORT_LIMIT = 0x103, NVME_SC_ABORT_MISSING = 0x104, NVME_SC_ASYNC_LIMIT = 0x105, NVME_SC_FIRMWARE_SLOT = 0x106, NVME_SC_FIRMWARE_IMAGE = 0x107, NVME_SC_INVALID_VECTOR = 0x108, NVME_SC_INVALID_LOG_PAGE = 0x109, NVME_SC_INVALID_FORMAT = 0x10a, NVME_SC_FW_NEEDS_CONV_RESET = 0x10b, NVME_SC_INVALID_QUEUE = 0x10c, NVME_SC_FEATURE_NOT_SAVEABLE = 0x10d, NVME_SC_FEATURE_NOT_CHANGEABLE = 0x10e, NVME_SC_FEATURE_NOT_PER_NS = 0x10f, NVME_SC_FW_NEEDS_SUBSYS_RESET = 0x110, NVME_SC_FW_NEEDS_RESET = 0x111, NVME_SC_FW_NEEDS_MAX_TIME = 0x112, NVME_SC_FW_ACTIVATE_PROHIBITED = 0x113, NVME_SC_OVERLAPPING_RANGE = 0x114, NVME_SC_NS_INSUFFICIENT_CAP = 0x115, NVME_SC_NS_ID_UNAVAILABLE = 0x116, NVME_SC_NS_ALREADY_ATTACHED = 0x118, NVME_SC_NS_IS_PRIVATE = 0x119, NVME_SC_NS_NOT_ATTACHED = 0x11a, NVME_SC_THIN_PROV_NOT_SUPP = 0x11b, NVME_SC_CTRL_LIST_INVALID = 0x11c, NVME_SC_SELF_TEST_IN_PROGRESS = 0x11d, NVME_SC_BP_WRITE_PROHIBITED = 0x11e, NVME_SC_CTRL_ID_INVALID = 0x11f, NVME_SC_SEC_CTRL_STATE_INVALID = 0x120, NVME_SC_CTRL_RES_NUM_INVALID = 0x121, NVME_SC_RES_ID_INVALID = 0x122, NVME_SC_PMR_SAN_PROHIBITED = 0x123, NVME_SC_ANA_GROUP_ID_INVALID = 0x124, NVME_SC_ANA_ATTACH_FAILED = 0x125, /* * I/O Command Set Specific - NVM commands: */ NVME_SC_BAD_ATTRIBUTES = 0x180, NVME_SC_INVALID_PI = 0x181, NVME_SC_READ_ONLY = 0x182, NVME_SC_CMD_SIZE_LIM_EXCEEDED = 0x183, /* * I/O Command Set Specific - Fabrics commands: */ NVME_SC_CONNECT_FORMAT = 0x180, NVME_SC_CONNECT_CTRL_BUSY = 0x181, NVME_SC_CONNECT_INVALID_PARAM = 0x182, NVME_SC_CONNECT_RESTART_DISC = 0x183, NVME_SC_CONNECT_INVALID_HOST = 0x184, NVME_SC_DISCOVERY_RESTART = 0x190, NVME_SC_AUTH_REQUIRED = 0x191, /* * I/O Command Set Specific - Zoned commands: */ NVME_SC_ZONE_BOUNDARY_ERROR = 0x1b8, NVME_SC_ZONE_FULL = 0x1b9, NVME_SC_ZONE_READ_ONLY = 0x1ba, NVME_SC_ZONE_OFFLINE = 0x1bb, NVME_SC_ZONE_INVALID_WRITE = 0x1bc, NVME_SC_ZONE_TOO_MANY_ACTIVE = 0x1bd, NVME_SC_ZONE_TOO_MANY_OPEN = 0x1be, NVME_SC_ZONE_INVALID_TRANSITION = 0x1bf, /* * Media and Data Integrity Errors: */ NVME_SCT_MEDIA_ERROR = 0x200, NVME_SC_WRITE_FAULT = 0x280, NVME_SC_READ_ERROR = 0x281, NVME_SC_GUARD_CHECK = 0x282, NVME_SC_APPTAG_CHECK = 0x283, NVME_SC_REFTAG_CHECK = 0x284, NVME_SC_COMPARE_FAILED = 0x285, NVME_SC_ACCESS_DENIED = 0x286, NVME_SC_UNWRITTEN_BLOCK = 0x287, /* * Path-related Errors: */ NVME_SCT_PATH = 0x300, NVME_SC_INTERNAL_PATH_ERROR = 0x300, NVME_SC_ANA_PERSISTENT_LOSS = 0x301, NVME_SC_ANA_INACCESSIBLE = 0x302, NVME_SC_ANA_TRANSITION = 0x303, NVME_SC_CTRL_PATH_ERROR = 0x360, NVME_SC_HOST_PATH_ERROR = 0x370, NVME_SC_HOST_ABORTED_CMD = 0x371, NVME_SC_MASK = 0x00ff, /* Status Code */ NVME_SCT_MASK = 0x0700, /* Status Code Type */ NVME_SCT_SC_MASK = NVME_SCT_MASK | NVME_SC_MASK, NVME_STATUS_CRD = 0x1800, /* Command Retry Delayed */ NVME_STATUS_MORE = 0x2000, NVME_STATUS_DNR = 0x4000, /* Do Not Retry */ }; #define NVME_SCT(status) ((status) >> 8 & 7) struct nvme_completion { /* * Used by Admin and Fabrics commands to return data: */ union nvme_result { __le16 u16; __le32 u32; __le64 u64; } result; __le16 sq_head; /* how much of this queue may be reclaimed */ __le16 sq_id; /* submission queue that generated this entry */ __u16 command_id; /* of the command which completed */ __le16 status; /* did the command fail, and if so, why? */ }; #define NVME_VS(major, minor, tertiary) \ (((major) << 16) | ((minor) << 8) | (tertiary)) #define NVME_MAJOR(ver) ((ver) >> 16) #define NVME_MINOR(ver) (((ver) >> 8) & 0xff) #define NVME_TERTIARY(ver) ((ver) & 0xff) enum { NVME_AEN_RESV_LOG_PAGE_AVALIABLE = 0x00, }; enum { NVME_PR_LOG_EMPTY_LOG_PAGE = 0x00, NVME_PR_LOG_REGISTRATION_PREEMPTED = 0x01, NVME_PR_LOG_RESERVATION_RELEASED = 0x02, NVME_PR_LOG_RESERVATOIN_PREEMPTED = 0x03, }; enum { NVME_PR_NOTIFY_BIT_REG_PREEMPTED = 1, NVME_PR_NOTIFY_BIT_RESV_RELEASED = 2, NVME_PR_NOTIFY_BIT_RESV_PREEMPTED = 3, }; struct nvme_pr_log { __le64 count; __u8 type; __u8 nr_pages; __u8 rsvd1[2]; __le32 nsid; __u8 rsvd2[48]; }; struct nvmet_pr_register_data { __le64 crkey; __le64 nrkey; }; struct nvmet_pr_acquire_data { __le64 crkey; __le64 prkey; }; struct nvmet_pr_release_data { __le64 crkey; }; enum nvme_pr_capabilities { NVME_PR_SUPPORT_PTPL = 1, NVME_PR_SUPPORT_WRITE_EXCLUSIVE = 1 << 1, NVME_PR_SUPPORT_EXCLUSIVE_ACCESS = 1 << 2, NVME_PR_SUPPORT_WRITE_EXCLUSIVE_REG_ONLY = 1 << 3, NVME_PR_SUPPORT_EXCLUSIVE_ACCESS_REG_ONLY = 1 << 4, NVME_PR_SUPPORT_WRITE_EXCLUSIVE_ALL_REGS = 1 << 5, NVME_PR_SUPPORT_EXCLUSIVE_ACCESS_ALL_REGS = 1 << 6, NVME_PR_SUPPORT_IEKEY_VER_1_3_DEF = 1 << 7, }; enum nvme_pr_register_action { NVME_PR_REGISTER_ACT_REG = 0, NVME_PR_REGISTER_ACT_UNREG = 1, NVME_PR_REGISTER_ACT_REPLACE = 1 << 1, }; enum nvme_pr_acquire_action { NVME_PR_ACQUIRE_ACT_ACQUIRE = 0, NVME_PR_ACQUIRE_ACT_PREEMPT = 1, NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT = 1 << 1, }; enum nvme_pr_release_action { NVME_PR_RELEASE_ACT_RELEASE = 0, NVME_PR_RELEASE_ACT_CLEAR = 1, }; enum nvme_pr_change_ptpl { NVME_PR_CPTPL_NO_CHANGE = 0, NVME_PR_CPTPL_RESV = 1 << 30, NVME_PR_CPTPL_CLEARED = 2 << 30, NVME_PR_CPTPL_PERSIST = 3 << 30, }; #define NVME_PR_IGNORE_KEY (1 << 3) /* Section 8.3.4.5.2 of the NVMe 2.1 */ #define NVME_AUTH_DHCHAP_MAX_HASH_IDS 30 #define NVME_AUTH_DHCHAP_MAX_DH_IDS 30 #endif /* _LINUX_NVME_H */ |
| 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2004, 2005 Oracle. All rights reserved. */ #include "linux/kstrtox.h" #include <linux/kernel.h> #include <linux/sched.h> #include <linux/jiffies.h> #include <linux/module.h> #include <linux/fs.h> #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/delay.h> #include <linux/file.h> #include <linux/kthread.h> #include <linux/configfs.h> #include <linux/random.h> #include <linux/crc32.h> #include <linux/time.h> #include <linux/debugfs.h> #include <linux/slab.h> #include <linux/bitmap.h> #include <linux/ktime.h> #include "heartbeat.h" #include "tcp.h" #include "nodemanager.h" #include "quorum.h" #include "masklog.h" /* * The first heartbeat pass had one global thread that would serialize all hb * callback calls. This global serializing sem should only be removed once * we've made sure that all callees can deal with being called concurrently * from multiple hb region threads. */ static DECLARE_RWSEM(o2hb_callback_sem); /* * multiple hb threads are watching multiple regions. A node is live * whenever any of the threads sees activity from the node in its region. */ static DEFINE_SPINLOCK(o2hb_live_lock); static struct list_head o2hb_live_slots[O2NM_MAX_NODES]; static unsigned long o2hb_live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; static LIST_HEAD(o2hb_node_events); static DECLARE_WAIT_QUEUE_HEAD(o2hb_steady_queue); /* * In global heartbeat, we maintain a series of region bitmaps. * - o2hb_region_bitmap allows us to limit the region number to max region. * - o2hb_live_region_bitmap tracks live regions (seen steady iterations). * - o2hb_quorum_region_bitmap tracks live regions that have seen all nodes * heartbeat on it. * - o2hb_failed_region_bitmap tracks the regions that have seen io timeouts. */ static unsigned long o2hb_region_bitmap[BITS_TO_LONGS(O2NM_MAX_REGIONS)]; static unsigned long o2hb_live_region_bitmap[BITS_TO_LONGS(O2NM_MAX_REGIONS)]; static unsigned long o2hb_quorum_region_bitmap[BITS_TO_LONGS(O2NM_MAX_REGIONS)]; static unsigned long o2hb_failed_region_bitmap[BITS_TO_LONGS(O2NM_MAX_REGIONS)]; #define O2HB_DB_TYPE_LIVENODES 0 #define O2HB_DB_TYPE_LIVEREGIONS 1 #define O2HB_DB_TYPE_QUORUMREGIONS 2 #define O2HB_DB_TYPE_FAILEDREGIONS 3 #define O2HB_DB_TYPE_REGION_LIVENODES 4 #define O2HB_DB_TYPE_REGION_NUMBER 5 #define O2HB_DB_TYPE_REGION_ELAPSED_TIME 6 #define O2HB_DB_TYPE_REGION_PINNED 7 struct o2hb_debug_buf { int db_type; int db_size; int db_len; void *db_data; }; static struct o2hb_debug_buf *o2hb_db_livenodes; static struct o2hb_debug_buf *o2hb_db_liveregions; static struct o2hb_debug_buf *o2hb_db_quorumregions; static struct o2hb_debug_buf *o2hb_db_failedregions; #define O2HB_DEBUG_DIR "o2hb" #define O2HB_DEBUG_LIVENODES "livenodes" #define O2HB_DEBUG_LIVEREGIONS "live_regions" #define O2HB_DEBUG_QUORUMREGIONS "quorum_regions" #define O2HB_DEBUG_FAILEDREGIONS "failed_regions" #define O2HB_DEBUG_REGION_NUMBER "num" #define O2HB_DEBUG_REGION_ELAPSED_TIME "elapsed_time_in_ms" #define O2HB_DEBUG_REGION_PINNED "pinned" static struct dentry *o2hb_debug_dir; static LIST_HEAD(o2hb_all_regions); static struct o2hb_callback { struct list_head list; } o2hb_callbacks[O2HB_NUM_CB]; static struct o2hb_callback *hbcall_from_type(enum o2hb_callback_type type); enum o2hb_heartbeat_modes { O2HB_HEARTBEAT_LOCAL = 0, O2HB_HEARTBEAT_GLOBAL, O2HB_HEARTBEAT_NUM_MODES, }; static const char *o2hb_heartbeat_mode_desc[O2HB_HEARTBEAT_NUM_MODES] = { "local", /* O2HB_HEARTBEAT_LOCAL */ "global", /* O2HB_HEARTBEAT_GLOBAL */ }; unsigned int o2hb_dead_threshold = O2HB_DEFAULT_DEAD_THRESHOLD; static unsigned int o2hb_heartbeat_mode = O2HB_HEARTBEAT_LOCAL; /* * o2hb_dependent_users tracks the number of registered callbacks that depend * on heartbeat. o2net and o2dlm are two entities that register this callback. * However only o2dlm depends on the heartbeat. It does not want the heartbeat * to stop while a dlm domain is still active. */ static unsigned int o2hb_dependent_users; /* * In global heartbeat mode, all regions are pinned if there are one or more * dependent users and the quorum region count is <= O2HB_PIN_CUT_OFF. All * regions are unpinned if the region count exceeds the cut off or the number * of dependent users falls to zero. */ #define O2HB_PIN_CUT_OFF 3 /* * In local heartbeat mode, we assume the dlm domain name to be the same as * region uuid. This is true for domains created for the file system but not * necessarily true for userdlm domains. This is a known limitation. * * In global heartbeat mode, we pin/unpin all o2hb regions. This solution * works for both file system and userdlm domains. */ static int o2hb_region_pin(const char *region_uuid); static void o2hb_region_unpin(const char *region_uuid); /* Only sets a new threshold if there are no active regions. * * No locking or otherwise interesting code is required for reading * o2hb_dead_threshold as it can't change once regions are active and * it's not interesting to anyone until then anyway. */ static void o2hb_dead_threshold_set(unsigned int threshold) { if (threshold > O2HB_MIN_DEAD_THRESHOLD) { spin_lock(&o2hb_live_lock); if (list_empty(&o2hb_all_regions)) o2hb_dead_threshold = threshold; spin_unlock(&o2hb_live_lock); } } static int o2hb_global_heartbeat_mode_set(unsigned int hb_mode) { int ret = -1; if (hb_mode < O2HB_HEARTBEAT_NUM_MODES) { spin_lock(&o2hb_live_lock); if (list_empty(&o2hb_all_regions)) { o2hb_heartbeat_mode = hb_mode; ret = 0; } spin_unlock(&o2hb_live_lock); } return ret; } struct o2hb_node_event { struct list_head hn_item; enum o2hb_callback_type hn_event_type; struct o2nm_node *hn_node; int hn_node_num; }; struct o2hb_disk_slot { struct o2hb_disk_heartbeat_block *ds_raw_block; u8 ds_node_num; u64 ds_last_time; u64 ds_last_generation; u16 ds_equal_samples; u16 ds_changed_samples; struct list_head ds_live_item; }; /* each thread owns a region.. when we're asked to tear down the region * we ask the thread to stop, who cleans up the region */ struct o2hb_region { struct config_item hr_item; struct list_head hr_all_item; unsigned hr_unclean_stop:1, hr_aborted_start:1, hr_item_pinned:1, hr_item_dropped:1, hr_node_deleted:1; /* protected by the hr_callback_sem */ struct task_struct *hr_task; unsigned int hr_blocks; unsigned long long hr_start_block; unsigned int hr_block_bits; unsigned int hr_block_bytes; unsigned int hr_slots_per_page; unsigned int hr_num_pages; struct page **hr_slot_data; struct file *hr_bdev_file; struct o2hb_disk_slot *hr_slots; /* live node map of this region */ unsigned long hr_live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; unsigned int hr_region_num; struct dentry *hr_debug_dir; struct o2hb_debug_buf *hr_db_livenodes; struct o2hb_debug_buf *hr_db_regnum; struct o2hb_debug_buf *hr_db_elapsed_time; struct o2hb_debug_buf *hr_db_pinned; /* let the person setting up hb wait for it to return until it * has reached a 'steady' state. This will be fixed when we have * a more complete api that doesn't lead to this sort of fragility. */ atomic_t hr_steady_iterations; /* terminate o2hb thread if it does not reach steady state * (hr_steady_iterations == 0) within hr_unsteady_iterations */ atomic_t hr_unsteady_iterations; unsigned int hr_timeout_ms; /* randomized as the region goes up and down so that a node * recognizes a node going up and down in one iteration */ u64 hr_generation; struct delayed_work hr_write_timeout_work; unsigned long hr_last_timeout_start; /* negotiate timer, used to negotiate extending hb timeout. */ struct delayed_work hr_nego_timeout_work; unsigned long hr_nego_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; /* Used during o2hb_check_slot to hold a copy of the block * being checked because we temporarily have to zero out the * crc field. */ struct o2hb_disk_heartbeat_block *hr_tmp_block; /* Message key for negotiate timeout message. */ unsigned int hr_key; struct list_head hr_handler_list; /* last hb status, 0 for success, other value for error. */ int hr_last_hb_status; }; static inline struct block_device *reg_bdev(struct o2hb_region *reg) { return reg->hr_bdev_file ? file_bdev(reg->hr_bdev_file) : NULL; } struct o2hb_bio_wait_ctxt { atomic_t wc_num_reqs; struct completion wc_io_complete; int wc_error; }; #define O2HB_NEGO_TIMEOUT_MS (O2HB_MAX_WRITE_TIMEOUT_MS/2) enum { O2HB_NEGO_TIMEOUT_MSG = 1, O2HB_NEGO_APPROVE_MSG = 2, }; struct o2hb_nego_msg { u8 node_num; }; static void o2hb_write_timeout(struct work_struct *work) { int failed, quorum; struct o2hb_region *reg = container_of(work, struct o2hb_region, hr_write_timeout_work.work); mlog(ML_ERROR, "Heartbeat write timeout to device %pg after %u " "milliseconds\n", reg_bdev(reg), jiffies_to_msecs(jiffies - reg->hr_last_timeout_start)); if (o2hb_global_heartbeat_active()) { spin_lock(&o2hb_live_lock); if (test_bit(reg->hr_region_num, o2hb_quorum_region_bitmap)) set_bit(reg->hr_region_num, o2hb_failed_region_bitmap); failed = bitmap_weight(o2hb_failed_region_bitmap, O2NM_MAX_REGIONS); quorum = bitmap_weight(o2hb_quorum_region_bitmap, O2NM_MAX_REGIONS); spin_unlock(&o2hb_live_lock); mlog(ML_HEARTBEAT, "Number of regions %d, failed regions %d\n", quorum, failed); /* * Fence if the number of failed regions >= half the number * of quorum regions */ if ((failed << 1) < quorum) return; } o2quo_disk_timeout(); } static void o2hb_arm_timeout(struct o2hb_region *reg) { /* Arm writeout only after thread reaches steady state */ if (atomic_read(®->hr_steady_iterations) != 0) return; mlog(ML_HEARTBEAT, "Queue write timeout for %u ms\n", O2HB_MAX_WRITE_TIMEOUT_MS); if (o2hb_global_heartbeat_active()) { spin_lock(&o2hb_live_lock); clear_bit(reg->hr_region_num, o2hb_failed_region_bitmap); spin_unlock(&o2hb_live_lock); } cancel_delayed_work(®->hr_write_timeout_work); schedule_delayed_work(®->hr_write_timeout_work, msecs_to_jiffies(O2HB_MAX_WRITE_TIMEOUT_MS)); cancel_delayed_work(®->hr_nego_timeout_work); /* negotiate timeout must be less than write timeout. */ schedule_delayed_work(®->hr_nego_timeout_work, msecs_to_jiffies(O2HB_NEGO_TIMEOUT_MS)); bitmap_zero(reg->hr_nego_node_bitmap, O2NM_MAX_NODES); } static void o2hb_disarm_timeout(struct o2hb_region *reg) { cancel_delayed_work_sync(®->hr_write_timeout_work); cancel_delayed_work_sync(®->hr_nego_timeout_work); } static int o2hb_send_nego_msg(int key, int type, u8 target) { struct o2hb_nego_msg msg; int status, ret; msg.node_num = o2nm_this_node(); again: ret = o2net_send_message(type, key, &msg, sizeof(msg), target, &status); if (ret == -EAGAIN || ret == -ENOMEM) { msleep(100); goto again; } return ret; } static void o2hb_nego_timeout(struct work_struct *work) { unsigned long live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; int master_node, i, ret; struct o2hb_region *reg; reg = container_of(work, struct o2hb_region, hr_nego_timeout_work.work); /* don't negotiate timeout if last hb failed since it is very * possible io failed. Should let write timeout fence self. */ if (reg->hr_last_hb_status) return; o2hb_fill_node_map(live_node_bitmap, O2NM_MAX_NODES); /* lowest node as master node to make negotiate decision. */ master_node = find_first_bit(live_node_bitmap, O2NM_MAX_NODES); if (master_node == o2nm_this_node()) { if (!test_bit(master_node, reg->hr_nego_node_bitmap)) { printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg).\n", o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000, config_item_name(®->hr_item), reg_bdev(reg)); set_bit(master_node, reg->hr_nego_node_bitmap); } if (!bitmap_equal(reg->hr_nego_node_bitmap, live_node_bitmap, O2NM_MAX_NODES)) { /* check negotiate bitmap every second to do timeout * approve decision. */ schedule_delayed_work(®->hr_nego_timeout_work, msecs_to_jiffies(1000)); return; } printk(KERN_NOTICE "o2hb: all nodes hb write hung, maybe region %s (%pg) is down.\n", config_item_name(®->hr_item), reg_bdev(reg)); /* approve negotiate timeout request. */ o2hb_arm_timeout(reg); i = -1; while ((i = find_next_bit(live_node_bitmap, O2NM_MAX_NODES, i + 1)) < O2NM_MAX_NODES) { if (i == master_node) continue; mlog(ML_HEARTBEAT, "send NEGO_APPROVE msg to node %d\n", i); ret = o2hb_send_nego_msg(reg->hr_key, O2HB_NEGO_APPROVE_MSG, i); if (ret) mlog(ML_ERROR, "send NEGO_APPROVE msg to node %d fail %d\n", i, ret); } } else { /* negotiate timeout with master node. */ printk(KERN_NOTICE "o2hb: node %d hb write hung for %ds on region %s (%pg), negotiate timeout with node %d.\n", o2nm_this_node(), O2HB_NEGO_TIMEOUT_MS/1000, config_item_name(®->hr_item), reg_bdev(reg), master_node); ret = o2hb_send_nego_msg(reg->hr_key, O2HB_NEGO_TIMEOUT_MSG, master_node); if (ret) mlog(ML_ERROR, "send NEGO_TIMEOUT msg to node %d fail %d\n", master_node, ret); } } static int o2hb_nego_timeout_handler(struct o2net_msg *msg, u32 len, void *data, void **ret_data) { struct o2hb_region *reg = data; struct o2hb_nego_msg *nego_msg; nego_msg = (struct o2hb_nego_msg *)msg->buf; printk(KERN_NOTICE "o2hb: receive negotiate timeout message from node %d on region %s (%pg).\n", nego_msg->node_num, config_item_name(®->hr_item), reg_bdev(reg)); if (nego_msg->node_num < O2NM_MAX_NODES) set_bit(nego_msg->node_num, reg->hr_nego_node_bitmap); else mlog(ML_ERROR, "got nego timeout message from bad node.\n"); return 0; } static int o2hb_nego_approve_handler(struct o2net_msg *msg, u32 len, void *data, void **ret_data) { struct o2hb_region *reg = data; printk(KERN_NOTICE "o2hb: negotiate timeout approved by master node on region %s (%pg).\n", config_item_name(®->hr_item), reg_bdev(reg)); o2hb_arm_timeout(reg); return 0; } static inline void o2hb_bio_wait_init(struct o2hb_bio_wait_ctxt *wc) { atomic_set(&wc->wc_num_reqs, 1); init_completion(&wc->wc_io_complete); wc->wc_error = 0; } /* Used in error paths too */ static inline void o2hb_bio_wait_dec(struct o2hb_bio_wait_ctxt *wc, unsigned int num) { /* sadly atomic_sub_and_test() isn't available on all platforms. The * good news is that the fast path only completes one at a time */ while(num--) { if (atomic_dec_and_test(&wc->wc_num_reqs)) { BUG_ON(num > 0); complete(&wc->wc_io_complete); } } } static void o2hb_wait_on_io(struct o2hb_bio_wait_ctxt *wc) { o2hb_bio_wait_dec(wc, 1); wait_for_completion(&wc->wc_io_complete); } static void o2hb_bio_end_io(struct bio *bio) { struct o2hb_bio_wait_ctxt *wc = bio->bi_private; if (bio->bi_status) { mlog(ML_ERROR, "IO Error %d\n", bio->bi_status); wc->wc_error = blk_status_to_errno(bio->bi_status); } o2hb_bio_wait_dec(wc, 1); bio_put(bio); } /* Setup a Bio to cover I/O against num_slots slots starting at * start_slot. */ static struct bio *o2hb_setup_one_bio(struct o2hb_region *reg, struct o2hb_bio_wait_ctxt *wc, unsigned int *current_slot, unsigned int max_slots, blk_opf_t opf) { int len, current_page; unsigned int vec_len, vec_start; unsigned int bits = reg->hr_block_bits; unsigned int spp = reg->hr_slots_per_page; unsigned int cs = *current_slot; struct bio *bio; struct page *page; /* Testing has shown this allocation to take long enough under * GFP_KERNEL that the local node can get fenced. It would be * nicest if we could pre-allocate these bios and avoid this * all together. */ bio = bio_alloc(reg_bdev(reg), 16, opf, GFP_ATOMIC); if (!bio) { mlog(ML_ERROR, "Could not alloc slots BIO!\n"); bio = ERR_PTR(-ENOMEM); goto bail; } /* Must put everything in 512 byte sectors for the bio... */ bio->bi_iter.bi_sector = (reg->hr_start_block + cs) << (bits - 9); bio->bi_private = wc; bio->bi_end_io = o2hb_bio_end_io; vec_start = (cs << bits) % PAGE_SIZE; while(cs < max_slots) { current_page = cs / spp; page = reg->hr_slot_data[current_page]; vec_len = min(PAGE_SIZE - vec_start, (max_slots-cs) * (PAGE_SIZE/spp) ); mlog(ML_HB_BIO, "page %d, vec_len = %u, vec_start = %u\n", current_page, vec_len, vec_start); len = bio_add_page(bio, page, vec_len, vec_start); if (len != vec_len) break; cs += vec_len / (PAGE_SIZE/spp); vec_start = 0; } bail: *current_slot = cs; return bio; } static int o2hb_read_slots(struct o2hb_region *reg, unsigned int begin_slot, unsigned int max_slots) { unsigned int current_slot = begin_slot; int status; struct o2hb_bio_wait_ctxt wc; struct bio *bio; o2hb_bio_wait_init(&wc); while(current_slot < max_slots) { bio = o2hb_setup_one_bio(reg, &wc, ¤t_slot, max_slots, REQ_OP_READ); if (IS_ERR(bio)) { status = PTR_ERR(bio); mlog_errno(status); goto bail_and_wait; } atomic_inc(&wc.wc_num_reqs); submit_bio(bio); } status = 0; bail_and_wait: o2hb_wait_on_io(&wc); if (wc.wc_error && !status) status = wc.wc_error; return status; } static int o2hb_issue_node_write(struct o2hb_region *reg, struct o2hb_bio_wait_ctxt *write_wc) { int status; unsigned int slot; struct bio *bio; o2hb_bio_wait_init(write_wc); slot = o2nm_this_node(); bio = o2hb_setup_one_bio(reg, write_wc, &slot, slot+1, REQ_OP_WRITE | REQ_SYNC); if (IS_ERR(bio)) { status = PTR_ERR(bio); mlog_errno(status); goto bail; } atomic_inc(&write_wc->wc_num_reqs); submit_bio(bio); status = 0; bail: return status; } static u32 o2hb_compute_block_crc_le(struct o2hb_region *reg, struct o2hb_disk_heartbeat_block *hb_block) { __le32 old_cksum; u32 ret; /* We want to compute the block crc with a 0 value in the * hb_cksum field. Save it off here and replace after the * crc. */ old_cksum = hb_block->hb_cksum; hb_block->hb_cksum = 0; ret = crc32_le(0, (unsigned char *) hb_block, reg->hr_block_bytes); hb_block->hb_cksum = old_cksum; return ret; } static void o2hb_dump_slot(struct o2hb_disk_heartbeat_block *hb_block) { mlog(ML_ERROR, "Dump slot information: seq = 0x%llx, node = %u, " "cksum = 0x%x, generation 0x%llx\n", (long long)le64_to_cpu(hb_block->hb_seq), hb_block->hb_node, le32_to_cpu(hb_block->hb_cksum), (long long)le64_to_cpu(hb_block->hb_generation)); } static int o2hb_verify_crc(struct o2hb_region *reg, struct o2hb_disk_heartbeat_block *hb_block) { u32 read, computed; read = le32_to_cpu(hb_block->hb_cksum); computed = o2hb_compute_block_crc_le(reg, hb_block); return read == computed; } /* * Compare the slot data with what we wrote in the last iteration. * If the match fails, print an appropriate error message. This is to * detect errors like... another node hearting on the same slot, * flaky device that is losing writes, etc. * Returns 1 if check succeeds, 0 otherwise. */ static int o2hb_check_own_slot(struct o2hb_region *reg) { struct o2hb_disk_slot *slot; struct o2hb_disk_heartbeat_block *hb_block; char *errstr; slot = ®->hr_slots[o2nm_this_node()]; /* Don't check on our 1st timestamp */ if (!slot->ds_last_time) return 0; hb_block = slot->ds_raw_block; if (le64_to_cpu(hb_block->hb_seq) == slot->ds_last_time && le64_to_cpu(hb_block->hb_generation) == slot->ds_last_generation && hb_block->hb_node == slot->ds_node_num) return 1; #define ERRSTR1 "Another node is heartbeating on device" #define ERRSTR2 "Heartbeat generation mismatch on device" #define ERRSTR3 "Heartbeat sequence mismatch on device" if (hb_block->hb_node != slot->ds_node_num) errstr = ERRSTR1; else if (le64_to_cpu(hb_block->hb_generation) != slot->ds_last_generation) errstr = ERRSTR2; else errstr = ERRSTR3; mlog(ML_ERROR, "%s (%pg): expected(%u:0x%llx, 0x%llx), " "ondisk(%u:0x%llx, 0x%llx)\n", errstr, reg_bdev(reg), slot->ds_node_num, (unsigned long long)slot->ds_last_generation, (unsigned long long)slot->ds_last_time, hb_block->hb_node, (unsigned long long)le64_to_cpu(hb_block->hb_generation), (unsigned long long)le64_to_cpu(hb_block->hb_seq)); return 0; } static inline void o2hb_prepare_block(struct o2hb_region *reg, u64 generation) { int node_num; u64 cputime; struct o2hb_disk_slot *slot; struct o2hb_disk_heartbeat_block *hb_block; node_num = o2nm_this_node(); slot = ®->hr_slots[node_num]; hb_block = (struct o2hb_disk_heartbeat_block *)slot->ds_raw_block; memset(hb_block, 0, reg->hr_block_bytes); /* TODO: time stuff */ cputime = ktime_get_real_seconds(); if (!cputime) cputime = 1; hb_block->hb_seq = cpu_to_le64(cputime); hb_block->hb_node = node_num; hb_block->hb_generation = cpu_to_le64(generation); hb_block->hb_dead_ms = cpu_to_le32(o2hb_dead_threshold * O2HB_REGION_TIMEOUT_MS); /* This step must always happen last! */ hb_block->hb_cksum = cpu_to_le32(o2hb_compute_block_crc_le(reg, hb_block)); mlog(ML_HB_BIO, "our node generation = 0x%llx, cksum = 0x%x\n", (long long)generation, le32_to_cpu(hb_block->hb_cksum)); } static void o2hb_fire_callbacks(struct o2hb_callback *hbcall, struct o2nm_node *node, int idx) { struct o2hb_callback_func *f; list_for_each_entry(f, &hbcall->list, hc_item) { mlog(ML_HEARTBEAT, "calling funcs %p\n", f); (f->hc_func)(node, idx, f->hc_data); } } /* Will run the list in order until we process the passed event */ static void o2hb_run_event_list(struct o2hb_node_event *queued_event) { struct o2hb_callback *hbcall; struct o2hb_node_event *event; /* Holding callback sem assures we don't alter the callback * lists when doing this, and serializes ourselves with other * processes wanting callbacks. */ down_write(&o2hb_callback_sem); spin_lock(&o2hb_live_lock); while (!list_empty(&o2hb_node_events) && !list_empty(&queued_event->hn_item)) { event = list_entry(o2hb_node_events.next, struct o2hb_node_event, hn_item); list_del_init(&event->hn_item); spin_unlock(&o2hb_live_lock); mlog(ML_HEARTBEAT, "Node %s event for %d\n", event->hn_event_type == O2HB_NODE_UP_CB ? "UP" : "DOWN", event->hn_node_num); hbcall = hbcall_from_type(event->hn_event_type); /* We should *never* have gotten on to the list with a * bad type... This isn't something that we should try * to recover from. */ BUG_ON(IS_ERR(hbcall)); o2hb_fire_callbacks(hbcall, event->hn_node, event->hn_node_num); spin_lock(&o2hb_live_lock); } spin_unlock(&o2hb_live_lock); up_write(&o2hb_callback_sem); } static void o2hb_queue_node_event(struct o2hb_node_event *event, enum o2hb_callback_type type, struct o2nm_node *node, int node_num) { assert_spin_locked(&o2hb_live_lock); BUG_ON((!node) && (type != O2HB_NODE_DOWN_CB)); event->hn_event_type = type; event->hn_node = node; event->hn_node_num = node_num; mlog(ML_HEARTBEAT, "Queue node %s event for node %d\n", type == O2HB_NODE_UP_CB ? "UP" : "DOWN", node_num); list_add_tail(&event->hn_item, &o2hb_node_events); } static void o2hb_shutdown_slot(struct o2hb_disk_slot *slot) { struct o2hb_node_event event = { .hn_item = LIST_HEAD_INIT(event.hn_item), }; struct o2nm_node *node; int queued = 0; node = o2nm_get_node_by_num(slot->ds_node_num); if (!node) return; spin_lock(&o2hb_live_lock); if (!list_empty(&slot->ds_live_item)) { mlog(ML_HEARTBEAT, "Shutdown, node %d leaves region\n", slot->ds_node_num); list_del_init(&slot->ds_live_item); if (list_empty(&o2hb_live_slots[slot->ds_node_num])) { clear_bit(slot->ds_node_num, o2hb_live_node_bitmap); o2hb_queue_node_event(&event, O2HB_NODE_DOWN_CB, node, slot->ds_node_num); queued = 1; } } spin_unlock(&o2hb_live_lock); if (queued) o2hb_run_event_list(&event); o2nm_node_put(node); } static void o2hb_set_quorum_device(struct o2hb_region *reg) { if (!o2hb_global_heartbeat_active()) return; /* Prevent race with o2hb_heartbeat_group_drop_item() */ if (kthread_should_stop()) return; /* Tag region as quorum only after thread reaches steady state */ if (atomic_read(®->hr_steady_iterations) != 0) return; spin_lock(&o2hb_live_lock); if (test_bit(reg->hr_region_num, o2hb_quorum_region_bitmap)) goto unlock; /* * A region can be added to the quorum only when it sees all * live nodes heartbeat on it. In other words, the region has been * added to all nodes. */ if (!bitmap_equal(reg->hr_live_node_bitmap, o2hb_live_node_bitmap, O2NM_MAX_NODES)) goto unlock; printk(KERN_NOTICE "o2hb: Region %s (%pg) is now a quorum device\n", config_item_name(®->hr_item), reg_bdev(reg)); set_bit(reg->hr_region_num, o2hb_quorum_region_bitmap); /* * If global heartbeat active, unpin all regions if the * region count > CUT_OFF */ if (bitmap_weight(o2hb_quorum_region_bitmap, O2NM_MAX_REGIONS) > O2HB_PIN_CUT_OFF) o2hb_region_unpin(NULL); unlock: spin_unlock(&o2hb_live_lock); } static int o2hb_check_slot(struct o2hb_region *reg, struct o2hb_disk_slot *slot) { int changed = 0, gen_changed = 0; struct o2hb_node_event event = { .hn_item = LIST_HEAD_INIT(event.hn_item), }; struct o2nm_node *node; struct o2hb_disk_heartbeat_block *hb_block = reg->hr_tmp_block; u64 cputime; unsigned int dead_ms = o2hb_dead_threshold * O2HB_REGION_TIMEOUT_MS; unsigned int slot_dead_ms; int tmp; int queued = 0; memcpy(hb_block, slot->ds_raw_block, reg->hr_block_bytes); /* * If a node is no longer configured but is still in the livemap, we * may need to clear that bit from the livemap. */ node = o2nm_get_node_by_num(slot->ds_node_num); if (!node) { spin_lock(&o2hb_live_lock); tmp = test_bit(slot->ds_node_num, o2hb_live_node_bitmap); spin_unlock(&o2hb_live_lock); if (!tmp) return 0; } if (!o2hb_verify_crc(reg, hb_block)) { /* all paths from here will drop o2hb_live_lock for * us. */ spin_lock(&o2hb_live_lock); /* Don't print an error on the console in this case - * a freshly formatted heartbeat area will not have a * crc set on it. */ if (list_empty(&slot->ds_live_item)) goto out; /* The node is live but pushed out a bad crc. We * consider it a transient miss but don't populate any * other values as they may be junk. */ mlog(ML_ERROR, "Node %d has written a bad crc to %pg\n", slot->ds_node_num, reg_bdev(reg)); o2hb_dump_slot(hb_block); slot->ds_equal_samples++; goto fire_callbacks; } /* we don't care if these wrap.. the state transitions below * clear at the right places */ cputime = le64_to_cpu(hb_block->hb_seq); if (slot->ds_last_time != cputime) slot->ds_changed_samples++; else slot->ds_equal_samples++; slot->ds_last_time = cputime; /* The node changed heartbeat generations. We assume this to * mean it dropped off but came back before we timed out. We * want to consider it down for the time being but don't want * to lose any changed_samples state we might build up to * considering it live again. */ if (slot->ds_last_generation != le64_to_cpu(hb_block->hb_generation)) { gen_changed = 1; slot->ds_equal_samples = 0; mlog(ML_HEARTBEAT, "Node %d changed generation (0x%llx " "to 0x%llx)\n", slot->ds_node_num, (long long)slot->ds_last_generation, (long long)le64_to_cpu(hb_block->hb_generation)); } slot->ds_last_generation = le64_to_cpu(hb_block->hb_generation); mlog(ML_HEARTBEAT, "Slot %d gen 0x%llx cksum 0x%x " "seq %llu last %llu changed %u equal %u\n", slot->ds_node_num, (long long)slot->ds_last_generation, le32_to_cpu(hb_block->hb_cksum), (unsigned long long)le64_to_cpu(hb_block->hb_seq), (unsigned long long)slot->ds_last_time, slot->ds_changed_samples, slot->ds_equal_samples); spin_lock(&o2hb_live_lock); fire_callbacks: /* dead nodes only come to life after some number of * changes at any time during their dead time */ if (list_empty(&slot->ds_live_item) && slot->ds_changed_samples >= O2HB_LIVE_THRESHOLD) { mlog(ML_HEARTBEAT, "Node %d (id 0x%llx) joined my region\n", slot->ds_node_num, (long long)slot->ds_last_generation); set_bit(slot->ds_node_num, reg->hr_live_node_bitmap); /* first on the list generates a callback */ if (list_empty(&o2hb_live_slots[slot->ds_node_num])) { mlog(ML_HEARTBEAT, "o2hb: Add node %d to live nodes " "bitmap\n", slot->ds_node_num); set_bit(slot->ds_node_num, o2hb_live_node_bitmap); o2hb_queue_node_event(&event, O2HB_NODE_UP_CB, node, slot->ds_node_num); changed = 1; queued = 1; } list_add_tail(&slot->ds_live_item, &o2hb_live_slots[slot->ds_node_num]); slot->ds_equal_samples = 0; /* We want to be sure that all nodes agree on the * number of milliseconds before a node will be * considered dead. The self-fencing timeout is * computed from this value, and a discrepancy might * result in heartbeat calling a node dead when it * hasn't self-fenced yet. */ slot_dead_ms = le32_to_cpu(hb_block->hb_dead_ms); if (slot_dead_ms && slot_dead_ms != dead_ms) { /* TODO: Perhaps we can fail the region here. */ mlog(ML_ERROR, "Node %d on device %pg has a dead count " "of %u ms, but our count is %u ms.\n" "Please double check your configuration values " "for 'O2CB_HEARTBEAT_THRESHOLD'\n", slot->ds_node_num, reg_bdev(reg), slot_dead_ms, dead_ms); } goto out; } /* if the list is dead, we're done.. */ if (list_empty(&slot->ds_live_item)) goto out; /* live nodes only go dead after enough consecutive missed * samples.. reset the missed counter whenever we see * activity */ if (slot->ds_equal_samples >= o2hb_dead_threshold || gen_changed) { mlog(ML_HEARTBEAT, "Node %d left my region\n", slot->ds_node_num); clear_bit(slot->ds_node_num, reg->hr_live_node_bitmap); /* last off the live_slot generates a callback */ list_del_init(&slot->ds_live_item); if (list_empty(&o2hb_live_slots[slot->ds_node_num])) { mlog(ML_HEARTBEAT, "o2hb: Remove node %d from live " "nodes bitmap\n", slot->ds_node_num); clear_bit(slot->ds_node_num, o2hb_live_node_bitmap); /* node can be null */ o2hb_queue_node_event(&event, O2HB_NODE_DOWN_CB, node, slot->ds_node_num); changed = 1; queued = 1; } /* We don't clear this because the node is still * actually writing new blocks. */ if (!gen_changed) slot->ds_changed_samples = 0; goto out; } if (slot->ds_changed_samples) { slot->ds_changed_samples = 0; slot->ds_equal_samples = 0; } out: spin_unlock(&o2hb_live_lock); if (queued) o2hb_run_event_list(&event); if (node) o2nm_node_put(node); return changed; } static int o2hb_highest_node(unsigned long *nodes, int numbits) { return find_last_bit(nodes, numbits); } static int o2hb_lowest_node(unsigned long *nodes, int numbits) { return find_first_bit(nodes, numbits); } static int o2hb_do_disk_heartbeat(struct o2hb_region *reg) { int i, ret, highest_node, lowest_node; int membership_change = 0, own_slot_ok = 0; unsigned long configured_nodes[BITS_TO_LONGS(O2NM_MAX_NODES)]; unsigned long live_node_bitmap[BITS_TO_LONGS(O2NM_MAX_NODES)]; struct o2hb_bio_wait_ctxt write_wc; ret = o2nm_configured_node_map(configured_nodes, sizeof(configured_nodes)); if (ret) { mlog_errno(ret); goto bail; } /* * If a node is not configured but is in the livemap, we still need * to read the slot so as to be able to remove it from the livemap. */ o2hb_fill_node_map(live_node_bitmap, O2NM_MAX_NODES); i = -1; while ((i = find_next_bit(live_node_bitmap, O2NM_MAX_NODES, i + 1)) < O2NM_MAX_NODES) { set_bit(i, configured_nodes); } highest_node = o2hb_highest_node(configured_nodes, O2NM_MAX_NODES); lowest_node = o2hb_lowest_node(configured_nodes, O2NM_MAX_NODES); if (highest_node >= O2NM_MAX_NODES || lowest_node >= O2NM_MAX_NODES) { mlog(ML_NOTICE, "o2hb: No configured nodes found!\n"); ret = -EINVAL; goto bail; } /* No sense in reading the slots of nodes that don't exist * yet. Of course, if the node definitions have holes in them * then we're reading an empty slot anyway... Consider this * best-effort. */ ret = o2hb_read_slots(reg, lowest_node, highest_node + 1); if (ret < 0) { mlog_errno(ret); goto bail; } /* With an up to date view of the slots, we can check that no * other node has been improperly configured to heartbeat in * our slot. */ own_slot_ok = o2hb_check_own_slot(reg); /* fill in the proper info for our next heartbeat */ o2hb_prepare_block(reg, reg->hr_generation); ret = o2hb_issue_node_write(reg, &write_wc); if (ret < 0) { mlog_errno(ret); goto bail; } i = -1; while((i = find_next_bit(configured_nodes, O2NM_MAX_NODES, i + 1)) < O2NM_MAX_NODES) { membership_change |= o2hb_check_slot(reg, ®->hr_slots[i]); } /* * We have to be sure we've advertised ourselves on disk * before we can go to steady state. This ensures that * people we find in our steady state have seen us. */ o2hb_wait_on_io(&write_wc); if (write_wc.wc_error) { /* Do not re-arm the write timeout on I/O error - we * can't be sure that the new block ever made it to * disk */ mlog(ML_ERROR, "Write error %d on device \"%pg\"\n", write_wc.wc_error, reg_bdev(reg)); ret = write_wc.wc_error; goto bail; } /* Skip disarming the timeout if own slot has stale/bad data */ if (own_slot_ok) { o2hb_set_quorum_device(reg); o2hb_arm_timeout(reg); reg->hr_last_timeout_start = jiffies; } bail: /* let the person who launched us know when things are steady */ if (atomic_read(®->hr_steady_iterations) != 0) { if (!ret && own_slot_ok && !membership_change) { if (atomic_dec_and_test(®->hr_steady_iterations)) wake_up(&o2hb_steady_queue); } } if (atomic_read(®->hr_steady_iterations) != 0) { if (atomic_dec_and_test(®->hr_unsteady_iterations)) { printk(KERN_NOTICE "o2hb: Unable to stabilize " "heartbeat on region %s (%pg)\n", config_item_name(®->hr_item), reg_bdev(reg)); atomic_set(®->hr_steady_iterations, 0); reg->hr_aborted_start = 1; wake_up(&o2hb_steady_queue); ret = -EIO; } } return ret; } /* * we ride the region ref that the region dir holds. before the region * dir is removed and drops it ref it will wait to tear down this * thread. */ static int o2hb_thread(void *data) { int i, ret; struct o2hb_region *reg = data; struct o2hb_bio_wait_ctxt write_wc; ktime_t before_hb, after_hb; unsigned int elapsed_msec; mlog(ML_HEARTBEAT|ML_KTHREAD, "hb thread running\n"); set_user_nice(current, MIN_NICE); /* Pin node */ ret = o2nm_depend_this_node(); if (ret) { mlog(ML_ERROR, "Node has been deleted, ret = %d\n", ret); reg->hr_node_deleted = 1; wake_up(&o2hb_steady_queue); return 0; } while (!kthread_should_stop() && !reg->hr_unclean_stop && !reg->hr_aborted_start) { /* We track the time spent inside * o2hb_do_disk_heartbeat so that we avoid more than * hr_timeout_ms between disk writes. On busy systems * this should result in a heartbeat which is less * likely to time itself out. */ before_hb = ktime_get_real(); ret = o2hb_do_disk_heartbeat(reg); reg->hr_last_hb_status = ret; after_hb = ktime_get_real(); elapsed_msec = (unsigned int) ktime_ms_delta(after_hb, before_hb); mlog(ML_HEARTBEAT, "start = %lld, end = %lld, msec = %u, ret = %d\n", before_hb, after_hb, elapsed_msec, ret); if (!kthread_should_stop() && elapsed_msec < reg->hr_timeout_ms) { /* the kthread api has blocked signals for us so no * need to record the return value. */ msleep_interruptible(reg->hr_timeout_ms - elapsed_msec); } } o2hb_disarm_timeout(reg); /* unclean stop is only used in very bad situation */ for(i = 0; !reg->hr_unclean_stop && i < reg->hr_blocks; i++) o2hb_shutdown_slot(®->hr_slots[i]); /* Explicit down notification - avoid forcing the other nodes * to timeout on this region when we could just as easily * write a clear generation - thus indicating to them that * this node has left this region. */ if (!reg->hr_unclean_stop && !reg->hr_aborted_start) { o2hb_prepare_block(reg, 0); ret = o2hb_issue_node_write(reg, &write_wc); if (ret == 0) o2hb_wait_on_io(&write_wc); else mlog_errno(ret); } /* Unpin node */ o2nm_undepend_this_node(); mlog(ML_HEARTBEAT|ML_KTHREAD, "o2hb thread exiting\n"); return 0; } #ifdef CONFIG_DEBUG_FS static int o2hb_debug_open(struct inode *inode, struct file *file) { struct o2hb_debug_buf *db = inode->i_private; struct o2hb_region *reg; unsigned long map[BITS_TO_LONGS(O2NM_MAX_NODES)]; unsigned long lts; char *buf = NULL; int i = -1; int out = 0; /* max_nodes should be the largest bitmap we pass here */ BUG_ON(sizeof(map) < db->db_size); buf = kmalloc(PAGE_SIZE, GFP_KERNEL); if (!buf) goto bail; switch (db->db_type) { case O2HB_DB_TYPE_LIVENODES: case O2HB_DB_TYPE_LIVEREGIONS: case O2HB_DB_TYPE_QUORUMREGIONS: case O2HB_DB_TYPE_FAILEDREGIONS: spin_lock(&o2hb_live_lock); memcpy(map, db->db_data, db->db_size); spin_unlock(&o2hb_live_lock); break; case O2HB_DB_TYPE_REGION_LIVENODES: spin_lock(&o2hb_live_lock); reg = (struct o2hb_region *)db->db_data; memcpy(map, reg->hr_live_node_bitmap, db->db_size); spin_unlock(&o2hb_live_lock); break; case O2HB_DB_TYPE_REGION_NUMBER: reg = (struct o2hb_region *)db->db_data; out += scnprintf(buf + out, PAGE_SIZE - out, "%d\n", reg->hr_region_num); goto done; case O2HB_DB_TYPE_REGION_ELAPSED_TIME: reg = (struct o2hb_region *)db->db_data; lts = reg->hr_last_timeout_start; /* If 0, it has never been set before */ if (lts) lts = jiffies_to_msecs(jiffies - lts); out += scnprintf(buf + out, PAGE_SIZE - out, "%lu\n", lts); goto done; case O2HB_DB_TYPE_REGION_PINNED: reg = (struct o2hb_region *)db->db_data; out += scnprintf(buf + out, PAGE_SIZE - out, "%u\n", !!reg->hr_item_pinned); goto done; default: goto done; } while ((i = find_next_bit(map, db->db_len, i + 1)) < db->db_len) out += scnprintf(buf + out, PAGE_SIZE - out, "%d ", i); out += scnprintf(buf + out, PAGE_SIZE - out, "\n"); done: i_size_write(inode, out); file->private_data = buf; return 0; bail: return -ENOMEM; } static int o2hb_debug_release(struct inode *inode, struct file *file) { kfree(file->private_data); return 0; } static ssize_t o2hb_debug_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { return simple_read_from_buffer(buf, nbytes, ppos, file->private_data, i_size_read(file->f_mapping->host)); } #else static int o2hb_debug_open(struct inode *inode, struct file *file) { return 0; } static int o2hb_debug_release(struct inode *inode, struct file *file) { return 0; } static ssize_t o2hb_debug_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos) { return 0; } #endif /* CONFIG_DEBUG_FS */ static const struct file_operations o2hb_debug_fops = { .open = o2hb_debug_open, .release = o2hb_debug_release, .read = o2hb_debug_read, .llseek = generic_file_llseek, }; void o2hb_exit(void) { debugfs_remove_recursive(o2hb_debug_dir); kfree(o2hb_db_livenodes); kfree(o2hb_db_liveregions); kfree(o2hb_db_quorumregions); kfree(o2hb_db_failedregions); } static void o2hb_debug_create(const char *name, struct dentry *dir, struct o2hb_debug_buf **db, int db_len, int type, int size, int len, void *data) { *db = kmalloc(db_len, GFP_KERNEL); if (!*db) return; (*db)->db_type = type; (*db)->db_size = size; (*db)->db_len = len; (*db)->db_data = data; debugfs_create_file(name, S_IFREG|S_IRUSR, dir, *db, &o2hb_debug_fops); } static void o2hb_debug_init(void) { o2hb_debug_dir = debugfs_create_dir(O2HB_DEBUG_DIR, NULL); o2hb_debug_create(O2HB_DEBUG_LIVENODES, o2hb_debug_dir, &o2hb_db_livenodes, sizeof(*o2hb_db_livenodes), O2HB_DB_TYPE_LIVENODES, sizeof(o2hb_live_node_bitmap), O2NM_MAX_NODES, o2hb_live_node_bitmap); o2hb_debug_create(O2HB_DEBUG_LIVEREGIONS, o2hb_debug_dir, &o2hb_db_liveregions, sizeof(*o2hb_db_liveregions), O2HB_DB_TYPE_LIVEREGIONS, sizeof(o2hb_live_region_bitmap), O2NM_MAX_REGIONS, o2hb_live_region_bitmap); o2hb_debug_create(O2HB_DEBUG_QUORUMREGIONS, o2hb_debug_dir, &o2hb_db_quorumregions, sizeof(*o2hb_db_quorumregions), O2HB_DB_TYPE_QUORUMREGIONS, sizeof(o2hb_quorum_region_bitmap), O2NM_MAX_REGIONS, o2hb_quorum_region_bitmap); o2hb_debug_create(O2HB_DEBUG_FAILEDREGIONS, o2hb_debug_dir, &o2hb_db_failedregions, sizeof(*o2hb_db_failedregions), O2HB_DB_TYPE_FAILEDREGIONS, sizeof(o2hb_failed_region_bitmap), O2NM_MAX_REGIONS, o2hb_failed_region_bitmap); } void o2hb_init(void) { int i; for (i = 0; i < ARRAY_SIZE(o2hb_callbacks); i++) INIT_LIST_HEAD(&o2hb_callbacks[i].list); for (i = 0; i < ARRAY_SIZE(o2hb_live_slots); i++) INIT_LIST_HEAD(&o2hb_live_slots[i]); bitmap_zero(o2hb_live_node_bitmap, O2NM_MAX_NODES); bitmap_zero(o2hb_region_bitmap, O2NM_MAX_REGIONS); bitmap_zero(o2hb_live_region_bitmap, O2NM_MAX_REGIONS); bitmap_zero(o2hb_quorum_region_bitmap, O2NM_MAX_REGIONS); bitmap_zero(o2hb_failed_region_bitmap, O2NM_MAX_REGIONS); o2hb_dependent_users = 0; o2hb_debug_init(); } /* if we're already in a callback then we're already serialized by the sem */ static void o2hb_fill_node_map_from_callback(unsigned long *map, unsigned int bits) { bitmap_copy(map, o2hb_live_node_bitmap, bits); } /* * get a map of all nodes that are heartbeating in any regions */ void o2hb_fill_node_map(unsigned long *map, unsigned int bits) { /* callers want to serialize this map and callbacks so that they * can trust that they don't miss nodes coming to the party */ down_read(&o2hb_callback_sem); spin_lock(&o2hb_live_lock); o2hb_fill_node_map_from_callback(map, bits); spin_unlock(&o2hb_live_lock); up_read(&o2hb_callback_sem); } EXPORT_SYMBOL_GPL(o2hb_fill_node_map); /* * heartbeat configfs bits. The heartbeat set is a default set under * the cluster set in nodemanager.c. */ static struct o2hb_region *to_o2hb_region(struct config_item *item) { return item ? container_of(item, struct o2hb_region, hr_item) : NULL; } static void o2hb_unmap_slot_data(struct o2hb_region *reg) { int i; struct page *page; if (reg->hr_slot_data) { for (i = 0; i < reg->hr_num_pages; i++) { page = reg->hr_slot_data[i]; if (page) { __free_page(page); reg->hr_slot_data[i] = NULL; } } kfree(reg->hr_slot_data); reg->hr_slot_data = NULL; } kfree(reg->hr_slots); reg->hr_slots = NULL; kfree(reg->hr_tmp_block); reg->hr_tmp_block = NULL; } /* drop_item only drops its ref after killing the thread, nothing should * be using the region anymore. this has to clean up any state that * attributes might have built up. */ static void o2hb_region_release(struct config_item *item) { struct o2hb_region *reg = to_o2hb_region(item); mlog(ML_HEARTBEAT, "hb region release (%pg)\n", reg_bdev(reg)); o2hb_unmap_slot_data(reg); if (reg->hr_bdev_file) fput(reg->hr_bdev_file); debugfs_remove_recursive(reg->hr_debug_dir); kfree(reg->hr_db_livenodes); kfree(reg->hr_db_regnum); kfree(reg->hr_db_elapsed_time); kfree(reg->hr_db_pinned); spin_lock(&o2hb_live_lock); list_del(®->hr_all_item); spin_unlock(&o2hb_live_lock); o2net_unregister_handler_list(®->hr_handler_list); kfree(reg); } static int o2hb_read_block_input(struct o2hb_region *reg, const char *page, unsigned long *ret_bytes, unsigned int *ret_bits) { unsigned long bytes; char *p = (char *)page; int ret; ret = kstrtoul(p, 0, &bytes); if (ret) return ret; /* Heartbeat and fs min / max block sizes are the same. */ if (bytes > 4096 || bytes < 512) return -ERANGE; if (hweight16(bytes) != 1) return -EINVAL; if (ret_bytes) *ret_bytes = bytes; if (ret_bits) *ret_bits = ffs(bytes) - 1; return 0; } static ssize_t o2hb_region_block_bytes_show(struct config_item *item, char *page) { return sprintf(page, "%u\n", to_o2hb_region(item)->hr_block_bytes); } static ssize_t o2hb_region_block_bytes_store(struct config_item *item, const char *page, size_t count) { struct o2hb_region *reg = to_o2hb_region(item); int status; unsigned long block_bytes; unsigned int block_bits; if (reg->hr_bdev_file) return -EINVAL; status = o2hb_read_block_input(reg, page, &block_bytes, &block_bits); if (status) return status; reg->hr_block_bytes = (unsigned int)block_bytes; reg->hr_block_bits = block_bits; return count; } static ssize_t o2hb_region_start_block_show(struct config_item *item, char *page) { return sprintf(page, "%llu\n", to_o2hb_region(item)->hr_start_block); } static ssize_t o2hb_region_start_block_store(struct config_item *item, const char *page, size_t count) { struct o2hb_region *reg = to_o2hb_region(item); unsigned long long tmp; char *p = (char *)page; ssize_t ret; if (reg->hr_bdev_file) return -EINVAL; ret = kstrtoull(p, 0, &tmp); if (ret) return -EINVAL; reg->hr_start_block = tmp; return count; } static ssize_t o2hb_region_blocks_show(struct config_item *item, char *page) { return sprintf(page, "%d\n", to_o2hb_region(item)->hr_blocks); } static ssize_t o2hb_region_blocks_store(struct config_item *item, const char *page, size_t count) { struct o2hb_region *reg = to_o2hb_region(item); unsigned long tmp; char *p = (char *)page; int ret; if (reg->hr_bdev_file) return -EINVAL; ret = kstrtoul(p, 0, &tmp); if (ret) return ret; if (tmp > O2NM_MAX_NODES || tmp == 0) return -ERANGE; reg->hr_blocks = (unsigned int)tmp; return count; } static ssize_t o2hb_region_dev_show(struct config_item *item, char *page) { unsigned int ret = 0; if (to_o2hb_region(item)->hr_bdev_file) ret = sprintf(page, "%pg\n", reg_bdev(to_o2hb_region(item))); return ret; } static void o2hb_init_region_params(struct o2hb_region *reg) { reg->hr_slots_per_page = PAGE_SIZE >> reg->hr_block_bits; reg->hr_timeout_ms = O2HB_REGION_TIMEOUT_MS; mlog(ML_HEARTBEAT, "hr_start_block = %llu, hr_blocks = %u\n", reg->hr_start_block, reg->hr_blocks); mlog(ML_HEARTBEAT, "hr_block_bytes = %u, hr_block_bits = %u\n", reg->hr_block_bytes, reg->hr_block_bits); mlog(ML_HEARTBEAT, "hr_timeout_ms = %u\n", reg->hr_timeout_ms); mlog(ML_HEARTBEAT, "dead threshold = %u\n", o2hb_dead_threshold); } static int o2hb_map_slot_data(struct o2hb_region *reg) { int i, j; int ret = -ENOMEM; unsigned int last_slot; unsigned int spp = reg->hr_slots_per_page; struct page *page; char *raw; struct o2hb_disk_slot *slot; reg->hr_tmp_block = kmalloc(reg->hr_block_bytes, GFP_KERNEL); if (!reg->hr_tmp_block) goto out; reg->hr_slots = kzalloc_objs(struct o2hb_disk_slot, reg->hr_blocks); if (!reg->hr_slots) goto out; for (i = 0; i < reg->hr_blocks; i++) { slot = ®->hr_slots[i]; slot->ds_node_num = i; INIT_LIST_HEAD(&slot->ds_live_item); slot->ds_raw_block = NULL; } reg->hr_num_pages = (reg->hr_blocks + spp - 1) / spp; mlog(ML_HEARTBEAT, "Going to require %u pages to cover %u blocks " "at %u blocks per page\n", reg->hr_num_pages, reg->hr_blocks, spp); reg->hr_slot_data = kzalloc_objs(struct page *, reg->hr_num_pages); if (!reg->hr_slot_data) goto out; for (i = 0; i < reg->hr_num_pages; i++) { page = alloc_page(GFP_KERNEL); if (!page) goto out; reg->hr_slot_data[i] = page; last_slot = i * spp; raw = page_address(page); for (j = 0; (j < spp) && ((j + last_slot) < reg->hr_blocks); j++) { BUG_ON((j + last_slot) >= reg->hr_blocks); slot = ®->hr_slots[j + last_slot]; slot->ds_raw_block = (struct o2hb_disk_heartbeat_block *) raw; raw += reg->hr_block_bytes; } } return 0; out: o2hb_unmap_slot_data(reg); return ret; } /* Read in all the slots available and populate the tracking * structures so that we can start with a baseline idea of what's * there. */ static int o2hb_populate_slot_data(struct o2hb_region *reg) { int ret, i; struct o2hb_disk_slot *slot; struct o2hb_disk_heartbeat_block *hb_block; ret = o2hb_read_slots(reg, 0, reg->hr_blocks); if (ret) goto out; /* We only want to get an idea of the values initially in each * slot, so we do no verification - o2hb_check_slot will * actually determine if each configured slot is valid and * whether any values have changed. */ for(i = 0; i < reg->hr_blocks; i++) { slot = ®->hr_slots[i]; hb_block = (struct o2hb_disk_heartbeat_block *) slot->ds_raw_block; /* Only fill the values that o2hb_check_slot uses to * determine changing slots */ slot->ds_last_time = le64_to_cpu(hb_block->hb_seq); slot->ds_last_generation = le64_to_cpu(hb_block->hb_generation); } out: return ret; } /* * this is acting as commit; we set up all of hr_bdev_file and hr_task or * nothing */ static ssize_t o2hb_region_dev_store(struct config_item *item, const char *page, size_t count) { struct o2hb_region *reg = to_o2hb_region(item); struct task_struct *hb_task; long fd; int sectsize; char *p = (char *)page; ssize_t ret = -EINVAL; int live_threshold; if (reg->hr_bdev_file) return -EINVAL; /* We can't heartbeat without having had our node number * configured yet. */ if (o2nm_this_node() == O2NM_MAX_NODES) return -EINVAL; ret = kstrtol(p, 0, &fd); if (ret < 0) return -EINVAL; if (fd < 0 || fd >= INT_MAX) return -EINVAL; CLASS(fd, f)(fd); if (fd_empty(f)) return -EINVAL; if (reg->hr_blocks == 0 || reg->hr_start_block == 0 || reg->hr_block_bytes == 0) return -EINVAL; if (!S_ISBLK(fd_file(f)->f_mapping->host->i_mode)) return -EINVAL; reg->hr_bdev_file = bdev_file_open_by_dev(fd_file(f)->f_mapping->host->i_rdev, BLK_OPEN_WRITE | BLK_OPEN_READ, NULL, NULL); if (IS_ERR(reg->hr_bdev_file)) { ret = PTR_ERR(reg->hr_bdev_file); reg->hr_bdev_file = NULL; return ret; } sectsize = bdev_logical_block_size(reg_bdev(reg)); if (sectsize != reg->hr_block_bytes) { mlog(ML_ERROR, "blocksize %u incorrect for device, expected %d", reg->hr_block_bytes, sectsize); ret = -EINVAL; goto out; } reg->hr_aborted_start = 0; reg->hr_node_deleted = 0; o2hb_init_region_params(reg); /* Generation of zero is invalid */ do { get_random_bytes(®->hr_generation, sizeof(reg->hr_generation)); } while (reg->hr_generation == 0); ret = o2hb_map_slot_data(reg); if (ret) { mlog_errno(ret); goto out; } ret = o2hb_populate_slot_data(reg); if (ret) { mlog_errno(ret); goto out; } INIT_DELAYED_WORK(®->hr_write_timeout_work, o2hb_write_timeout); INIT_DELAYED_WORK(®->hr_nego_timeout_work, o2hb_nego_timeout); /* * A node is considered live after it has beat LIVE_THRESHOLD * times. We're not steady until we've given them a chance * _after_ our first read. * The default threshold is bare minimum so as to limit the delay * during mounts. For global heartbeat, the threshold doubled for the * first region. */ live_threshold = O2HB_LIVE_THRESHOLD; if (o2hb_global_heartbeat_active()) { spin_lock(&o2hb_live_lock); if (bitmap_weight(o2hb_region_bitmap, O2NM_MAX_REGIONS) == 1) live_threshold <<= 1; spin_unlock(&o2hb_live_lock); } ++live_threshold; atomic_set(®->hr_steady_iterations, live_threshold); /* unsteady_iterations is triple the steady_iterations */ atomic_set(®->hr_unsteady_iterations, (live_threshold * 3)); hb_task = kthread_run(o2hb_thread, reg, "o2hb-%s", reg->hr_item.ci_name); if (IS_ERR(hb_task)) { ret = PTR_ERR(hb_task); mlog_errno(ret); goto out; } spin_lock(&o2hb_live_lock); reg->hr_task = hb_task; spin_unlock(&o2hb_live_lock); ret = wait_event_interruptible(o2hb_steady_queue, atomic_read(®->hr_steady_iterations) == 0 || reg->hr_node_deleted); if (ret) { atomic_set(®->hr_steady_iterations, 0); reg->hr_aborted_start = 1; } if (reg->hr_aborted_start) { ret = -EIO; goto out; } if (reg->hr_node_deleted) { ret = -EINVAL; goto out; } /* Ok, we were woken. Make sure it wasn't by drop_item() */ spin_lock(&o2hb_live_lock); hb_task = reg->hr_task; if (o2hb_global_heartbeat_active()) set_bit(reg->hr_region_num, o2hb_live_region_bitmap); spin_unlock(&o2hb_live_lock); if (hb_task) ret = count; else ret = -EIO; if (hb_task && o2hb_global_heartbeat_active()) printk(KERN_NOTICE "o2hb: Heartbeat started on region %s (%pg)\n", config_item_name(®->hr_item), reg_bdev(reg)); out: if (ret < 0) { spin_lock(&o2hb_live_lock); hb_task = reg->hr_task; reg->hr_task = NULL; spin_unlock(&o2hb_live_lock); if (hb_task) kthread_stop(hb_task); o2hb_unmap_slot_data(reg); fput(reg->hr_bdev_file); reg->hr_bdev_file = NULL; } return ret; } static ssize_t o2hb_region_pid_show(struct config_item *item, char *page) { struct o2hb_region *reg = to_o2hb_region(item); pid_t pid = 0; spin_lock(&o2hb_live_lock); if (reg->hr_task) pid = task_pid_nr(reg->hr_task); spin_unlock(&o2hb_live_lock); if (!pid) return 0; return sprintf(page, "%u\n", pid); } CONFIGFS_ATTR(o2hb_region_, block_bytes); CONFIGFS_ATTR(o2hb_region_, start_block); CONFIGFS_ATTR(o2hb_region_, blocks); CONFIGFS_ATTR(o2hb_region_, dev); CONFIGFS_ATTR_RO(o2hb_region_, pid); static struct configfs_attribute *o2hb_region_attrs[] = { &o2hb_region_attr_block_bytes, &o2hb_region_attr_start_block, &o2hb_region_attr_blocks, &o2hb_region_attr_dev, &o2hb_region_attr_pid, NULL, }; static const struct configfs_item_operations o2hb_region_item_ops = { .release = o2hb_region_release, }; static const struct config_item_type o2hb_region_type = { .ct_item_ops = &o2hb_region_item_ops, .ct_attrs = o2hb_region_attrs, .ct_owner = THIS_MODULE, }; /* heartbeat set */ struct o2hb_heartbeat_group { struct config_group hs_group; /* some stuff? */ }; static struct o2hb_heartbeat_group *to_o2hb_heartbeat_group(struct config_group *group) { return group ? container_of(group, struct o2hb_heartbeat_group, hs_group) : NULL; } static void o2hb_debug_region_init(struct o2hb_region *reg, struct dentry *parent) { struct dentry *dir; dir = debugfs_create_dir(config_item_name(®->hr_item), parent); reg->hr_debug_dir = dir; o2hb_debug_create(O2HB_DEBUG_LIVENODES, dir, &(reg->hr_db_livenodes), sizeof(*(reg->hr_db_livenodes)), O2HB_DB_TYPE_REGION_LIVENODES, sizeof(reg->hr_live_node_bitmap), O2NM_MAX_NODES, reg); o2hb_debug_create(O2HB_DEBUG_REGION_NUMBER, dir, &(reg->hr_db_regnum), sizeof(*(reg->hr_db_regnum)), O2HB_DB_TYPE_REGION_NUMBER, 0, O2NM_MAX_NODES, reg); o2hb_debug_create(O2HB_DEBUG_REGION_ELAPSED_TIME, dir, &(reg->hr_db_elapsed_time), sizeof(*(reg->hr_db_elapsed_time)), O2HB_DB_TYPE_REGION_ELAPSED_TIME, 0, 0, reg); o2hb_debug_create(O2HB_DEBUG_REGION_PINNED, dir, &(reg->hr_db_pinned), sizeof(*(reg->hr_db_pinned)), O2HB_DB_TYPE_REGION_PINNED, 0, 0, reg); } static struct config_item *o2hb_heartbeat_group_make_item(struct config_group *group, const char *name) { struct o2hb_region *reg = NULL; int ret; reg = kzalloc_obj(struct o2hb_region); if (reg == NULL) return ERR_PTR(-ENOMEM); if (strlen(name) > O2HB_MAX_REGION_NAME_LEN) { ret = -ENAMETOOLONG; goto free; } spin_lock(&o2hb_live_lock); reg->hr_region_num = 0; if (o2hb_global_heartbeat_active()) { reg->hr_region_num = find_first_zero_bit(o2hb_region_bitmap, O2NM_MAX_REGIONS); if (reg->hr_region_num >= O2NM_MAX_REGIONS) { spin_unlock(&o2hb_live_lock); ret = -EFBIG; goto free; } set_bit(reg->hr_region_num, o2hb_region_bitmap); } list_add_tail(®->hr_all_item, &o2hb_all_regions); spin_unlock(&o2hb_live_lock); config_item_init_type_name(®->hr_item, name, &o2hb_region_type); /* this is the same way to generate msg key as dlm, for local heartbeat, * name is also the same, so make initial crc value different to avoid * message key conflict. */ reg->hr_key = crc32_le(reg->hr_region_num + O2NM_MAX_REGIONS, name, strlen(name)); INIT_LIST_HEAD(®->hr_handler_list); ret = o2net_register_handler(O2HB_NEGO_TIMEOUT_MSG, reg->hr_key, sizeof(struct o2hb_nego_msg), o2hb_nego_timeout_handler, reg, NULL, ®->hr_handler_list); if (ret) goto remove_item; ret = o2net_register_handler(O2HB_NEGO_APPROVE_MSG, reg->hr_key, sizeof(struct o2hb_nego_msg), o2hb_nego_approve_handler, reg, NULL, ®->hr_handler_list); if (ret) goto unregister_handler; o2hb_debug_region_init(reg, o2hb_debug_dir); return ®->hr_item; unregister_handler: o2net_unregister_handler_list(®->hr_handler_list); remove_item: spin_lock(&o2hb_live_lock); list_del(®->hr_all_item); if (o2hb_global_heartbeat_active()) clear_bit(reg->hr_region_num, o2hb_region_bitmap); spin_unlock(&o2hb_live_lock); free: kfree(reg); return ERR_PTR(ret); } static void o2hb_heartbeat_group_drop_item(struct config_group *group, struct config_item *item) { struct task_struct *hb_task; struct o2hb_region *reg = to_o2hb_region(item); int quorum_region = 0; /* stop the thread when the user removes the region dir */ spin_lock(&o2hb_live_lock); hb_task = reg->hr_task; reg->hr_task = NULL; reg->hr_item_dropped = 1; spin_unlock(&o2hb_live_lock); if (hb_task) kthread_stop(hb_task); if (o2hb_global_heartbeat_active()) { spin_lock(&o2hb_live_lock); clear_bit(reg->hr_region_num, o2hb_region_bitmap); clear_bit(reg->hr_region_num, o2hb_live_region_bitmap); if (test_bit(reg->hr_region_num, o2hb_quorum_region_bitmap)) quorum_region = 1; clear_bit(reg->hr_region_num, o2hb_quorum_region_bitmap); spin_unlock(&o2hb_live_lock); printk(KERN_NOTICE "o2hb: Heartbeat %s on region %s (%pg)\n", ((atomic_read(®->hr_steady_iterations) == 0) ? "stopped" : "start aborted"), config_item_name(item), reg_bdev(reg)); } /* * If we're racing a dev_write(), we need to wake them. They will * check reg->hr_task */ if (atomic_read(®->hr_steady_iterations) != 0) { reg->hr_aborted_start = 1; atomic_set(®->hr_steady_iterations, 0); wake_up(&o2hb_steady_queue); } config_item_put(item); if (!o2hb_global_heartbeat_active() || !quorum_region) return; /* * If global heartbeat active and there are dependent users, * pin all regions if quorum region count <= CUT_OFF */ spin_lock(&o2hb_live_lock); if (!o2hb_dependent_users) goto unlock; if (bitmap_weight(o2hb_quorum_region_bitmap, O2NM_MAX_REGIONS) <= O2HB_PIN_CUT_OFF) o2hb_region_pin(NULL); unlock: spin_unlock(&o2hb_live_lock); } static ssize_t o2hb_heartbeat_group_dead_threshold_show(struct config_item *item, char *page) { return sprintf(page, "%u\n", o2hb_dead_threshold); } static ssize_t o2hb_heartbeat_group_dead_threshold_store(struct config_item *item, const char *page, size_t count) { unsigned long tmp; char *p = (char *)page; int ret; ret = kstrtoul(p, 10, &tmp); if (ret) return ret; /* this will validate ranges for us. */ o2hb_dead_threshold_set((unsigned int) tmp); return count; } static ssize_t o2hb_heartbeat_group_mode_show(struct config_item *item, char *page) { return sprintf(page, "%s\n", o2hb_heartbeat_mode_desc[o2hb_heartbeat_mode]); } static ssize_t o2hb_heartbeat_group_mode_store(struct config_item *item, const char *page, size_t count) { unsigned int i; int ret; size_t len; len = (page[count - 1] == '\n') ? count - 1 : count; if (!len) return -EINVAL; for (i = 0; i < O2HB_HEARTBEAT_NUM_MODES; ++i) { if (strncasecmp(page, o2hb_heartbeat_mode_desc[i], len)) continue; ret = o2hb_global_heartbeat_mode_set(i); if (!ret) printk(KERN_NOTICE "o2hb: Heartbeat mode set to %s\n", o2hb_heartbeat_mode_desc[i]); return count; } return -EINVAL; } CONFIGFS_ATTR(o2hb_heartbeat_group_, dead_threshold); CONFIGFS_ATTR(o2hb_heartbeat_group_, mode); static struct configfs_attribute *o2hb_heartbeat_group_attrs[] = { &o2hb_heartbeat_group_attr_dead_threshold, &o2hb_heartbeat_group_attr_mode, NULL, }; static const struct configfs_group_operations o2hb_heartbeat_group_group_ops = { .make_item = o2hb_heartbeat_group_make_item, .drop_item = o2hb_heartbeat_group_drop_item, }; static const struct config_item_type o2hb_heartbeat_group_type = { .ct_group_ops = &o2hb_heartbeat_group_group_ops, .ct_attrs = o2hb_heartbeat_group_attrs, .ct_owner = THIS_MODULE, }; /* this is just here to avoid touching group in heartbeat.h which the * entire damn world #includes */ struct config_group *o2hb_alloc_hb_set(void) { struct o2hb_heartbeat_group *hs = NULL; struct config_group *ret = NULL; hs = kzalloc_obj(struct o2hb_heartbeat_group); if (hs == NULL) goto out; config_group_init_type_name(&hs->hs_group, "heartbeat", &o2hb_heartbeat_group_type); ret = &hs->hs_group; out: if (ret == NULL) kfree(hs); return ret; } void o2hb_free_hb_set(struct config_group *group) { struct o2hb_heartbeat_group *hs = to_o2hb_heartbeat_group(group); kfree(hs); } /* hb callback registration and issuing */ static struct o2hb_callback *hbcall_from_type(enum o2hb_callback_type type) { if (type == O2HB_NUM_CB) return ERR_PTR(-EINVAL); return &o2hb_callbacks[type]; } void o2hb_setup_callback(struct o2hb_callback_func *hc, enum o2hb_callback_type type, o2hb_cb_func *func, void *data, int priority) { INIT_LIST_HEAD(&hc->hc_item); hc->hc_func = func; hc->hc_data = data; hc->hc_priority = priority; hc->hc_type = type; hc->hc_magic = O2HB_CB_MAGIC; } EXPORT_SYMBOL_GPL(o2hb_setup_callback); /* * In local heartbeat mode, region_uuid passed matches the dlm domain name. * In global heartbeat mode, region_uuid passed is NULL. * * In local, we only pin the matching region. In global we pin all the active * regions. */ static int o2hb_region_pin(const char *region_uuid) { int ret = 0, found = 0; struct o2hb_region *reg; char *uuid; assert_spin_locked(&o2hb_live_lock); list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) { if (reg->hr_item_dropped) continue; uuid = config_item_name(®->hr_item); /* local heartbeat */ if (region_uuid) { if (strcmp(region_uuid, uuid)) continue; found = 1; } if (reg->hr_item_pinned || reg->hr_item_dropped) goto skip_pin; /* Ignore ENOENT only for local hb (userdlm domain) */ ret = o2nm_depend_item(®->hr_item); if (!ret) { mlog(ML_CLUSTER, "Pin region %s\n", uuid); reg->hr_item_pinned = 1; } else { if (ret == -ENOENT && found) ret = 0; else { mlog(ML_ERROR, "Pin region %s fails with %d\n", uuid, ret); break; } } skip_pin: if (found) break; } return ret; } /* * In local heartbeat mode, region_uuid passed matches the dlm domain name. * In global heartbeat mode, region_uuid passed is NULL. * * In local, we only unpin the matching region. In global we unpin all the * active regions. */ static void o2hb_region_unpin(const char *region_uuid) { struct o2hb_region *reg; char *uuid; int found = 0; assert_spin_locked(&o2hb_live_lock); list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) { if (reg->hr_item_dropped) continue; uuid = config_item_name(®->hr_item); if (region_uuid) { if (strcmp(region_uuid, uuid)) continue; found = 1; } if (reg->hr_item_pinned) { mlog(ML_CLUSTER, "Unpin region %s\n", uuid); o2nm_undepend_item(®->hr_item); reg->hr_item_pinned = 0; } if (found) break; } } static int o2hb_region_inc_user(const char *region_uuid) { int ret = 0; spin_lock(&o2hb_live_lock); /* local heartbeat */ if (!o2hb_global_heartbeat_active()) { ret = o2hb_region_pin(region_uuid); goto unlock; } /* * if global heartbeat active and this is the first dependent user, * pin all regions if quorum region count <= CUT_OFF */ o2hb_dependent_users++; if (o2hb_dependent_users > 1) goto unlock; if (bitmap_weight(o2hb_quorum_region_bitmap, O2NM_MAX_REGIONS) <= O2HB_PIN_CUT_OFF) ret = o2hb_region_pin(NULL); unlock: spin_unlock(&o2hb_live_lock); return ret; } static void o2hb_region_dec_user(const char *region_uuid) { spin_lock(&o2hb_live_lock); /* local heartbeat */ if (!o2hb_global_heartbeat_active()) { o2hb_region_unpin(region_uuid); goto unlock; } /* * if global heartbeat active and there are no dependent users, * unpin all quorum regions */ o2hb_dependent_users--; if (!o2hb_dependent_users) o2hb_region_unpin(NULL); unlock: spin_unlock(&o2hb_live_lock); } int o2hb_register_callback(const char *region_uuid, struct o2hb_callback_func *hc) { struct o2hb_callback_func *f; struct o2hb_callback *hbcall; int ret; BUG_ON(hc->hc_magic != O2HB_CB_MAGIC); BUG_ON(!list_empty(&hc->hc_item)); hbcall = hbcall_from_type(hc->hc_type); if (IS_ERR(hbcall)) { ret = PTR_ERR(hbcall); goto out; } if (region_uuid) { ret = o2hb_region_inc_user(region_uuid); if (ret) { mlog_errno(ret); goto out; } } down_write(&o2hb_callback_sem); list_for_each_entry(f, &hbcall->list, hc_item) { if (hc->hc_priority < f->hc_priority) { list_add_tail(&hc->hc_item, &f->hc_item); break; } } if (list_empty(&hc->hc_item)) list_add_tail(&hc->hc_item, &hbcall->list); up_write(&o2hb_callback_sem); ret = 0; out: mlog(ML_CLUSTER, "returning %d on behalf of %p for funcs %p\n", ret, __builtin_return_address(0), hc); return ret; } EXPORT_SYMBOL_GPL(o2hb_register_callback); void o2hb_unregister_callback(const char *region_uuid, struct o2hb_callback_func *hc) { BUG_ON(hc->hc_magic != O2HB_CB_MAGIC); mlog(ML_CLUSTER, "on behalf of %p for funcs %p\n", __builtin_return_address(0), hc); /* XXX Can this happen _with_ a region reference? */ if (list_empty(&hc->hc_item)) return; if (region_uuid) o2hb_region_dec_user(region_uuid); down_write(&o2hb_callback_sem); list_del_init(&hc->hc_item); up_write(&o2hb_callback_sem); } EXPORT_SYMBOL_GPL(o2hb_unregister_callback); int o2hb_check_node_heartbeating_no_sem(u8 node_num) { unsigned long testing_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; spin_lock(&o2hb_live_lock); o2hb_fill_node_map_from_callback(testing_map, O2NM_MAX_NODES); spin_unlock(&o2hb_live_lock); if (!test_bit(node_num, testing_map)) { mlog(ML_HEARTBEAT, "node (%u) does not have heartbeating enabled.\n", node_num); return 0; } return 1; } EXPORT_SYMBOL_GPL(o2hb_check_node_heartbeating_no_sem); int o2hb_check_node_heartbeating_from_callback(u8 node_num) { unsigned long testing_map[BITS_TO_LONGS(O2NM_MAX_NODES)]; o2hb_fill_node_map_from_callback(testing_map, O2NM_MAX_NODES); if (!test_bit(node_num, testing_map)) { mlog(ML_HEARTBEAT, "node (%u) does not have heartbeating enabled.\n", node_num); return 0; } return 1; } EXPORT_SYMBOL_GPL(o2hb_check_node_heartbeating_from_callback); /* * this is just a hack until we get the plumbing which flips file systems * read only and drops the hb ref instead of killing the node dead. */ void o2hb_stop_all_regions(void) { struct o2hb_region *reg; mlog(ML_ERROR, "stopping heartbeat on all active regions.\n"); spin_lock(&o2hb_live_lock); list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) reg->hr_unclean_stop = 1; spin_unlock(&o2hb_live_lock); } EXPORT_SYMBOL_GPL(o2hb_stop_all_regions); int o2hb_get_all_regions(char *region_uuids, u8 max_regions) { struct o2hb_region *reg; int numregs = 0; char *p; spin_lock(&o2hb_live_lock); p = region_uuids; list_for_each_entry(reg, &o2hb_all_regions, hr_all_item) { if (reg->hr_item_dropped) continue; mlog(0, "Region: %s\n", config_item_name(®->hr_item)); if (numregs < max_regions) { memcpy(p, config_item_name(®->hr_item), O2HB_MAX_REGION_NAME_LEN); p += O2HB_MAX_REGION_NAME_LEN; } numregs++; } spin_unlock(&o2hb_live_lock); return numregs; } EXPORT_SYMBOL_GPL(o2hb_get_all_regions); int o2hb_global_heartbeat_active(void) { return (o2hb_heartbeat_mode == O2HB_HEARTBEAT_GLOBAL); } EXPORT_SYMBOL(o2hb_global_heartbeat_active); |
| 115 115 115 104 94 94 115 21 97 85 98 95 98 1 1 1 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 | // SPDX-License-Identifier: GPL-2.0 /* * USB Serial Converter Bus specific functions * * Copyright (C) 2002 Greg Kroah-Hartman (greg@kroah.com) */ #include <linux/kernel.h> #include <linux/errno.h> #include <linux/tty.h> #include <linux/slab.h> #include <linux/module.h> #include <linux/usb.h> #include <linux/usb/serial.h> static int usb_serial_device_match(struct device *dev, const struct device_driver *drv) { const struct usb_serial_port *port = to_usb_serial_port(dev); const struct usb_serial_driver *driver = to_usb_serial_driver(drv); /* * drivers are already assigned to ports in serial_probe so it's * a simple check here. */ if (driver == port->serial->type) return 1; return 0; } static int usb_serial_device_probe(struct device *dev) { struct usb_serial_port *port = to_usb_serial_port(dev); struct usb_serial_driver *driver; struct device *tty_dev; int retval = 0; int minor; /* make sure suspend/resume doesn't race against port_probe */ retval = usb_autopm_get_interface(port->serial->interface); if (retval) return retval; driver = port->serial->type; if (driver->port_probe) { retval = driver->port_probe(port); if (retval) goto err_autopm_put; } minor = port->minor; tty_dev = tty_port_register_device(&port->port, usb_serial_tty_driver, minor, dev); if (IS_ERR(tty_dev)) { retval = PTR_ERR(tty_dev); goto err_port_remove; } usb_autopm_put_interface(port->serial->interface); dev_info(&port->serial->dev->dev, "%s converter now attached to ttyUSB%d\n", driver->description, minor); return 0; err_port_remove: if (driver->port_remove) driver->port_remove(port); err_autopm_put: usb_autopm_put_interface(port->serial->interface); return retval; } static void usb_serial_device_remove(struct device *dev) { struct usb_serial_port *port = to_usb_serial_port(dev); struct usb_serial_driver *driver; int minor; int autopm_err; /* * Make sure suspend/resume doesn't race against port_remove. * * Note that no further runtime PM callbacks will be made if * autopm_get fails. */ autopm_err = usb_autopm_get_interface(port->serial->interface); minor = port->minor; tty_unregister_device(usb_serial_tty_driver, minor); driver = port->serial->type; if (driver->port_remove) driver->port_remove(port); dev_info(dev, "%s converter now disconnected from ttyUSB%d\n", driver->description, minor); if (!autopm_err) usb_autopm_put_interface(port->serial->interface); } static ssize_t new_id_store(struct device_driver *driver, const char *buf, size_t count) { struct usb_serial_driver *usb_drv = to_usb_serial_driver(driver); ssize_t retval = usb_store_new_id(&usb_drv->dynids, usb_drv->id_table, driver, buf, count); if (retval >= 0 && usb_drv->usb_driver != NULL) retval = usb_store_new_id(&usb_drv->usb_driver->dynids, usb_drv->usb_driver->id_table, &usb_drv->usb_driver->driver, buf, count); return retval; } static ssize_t new_id_show(struct device_driver *driver, char *buf) { struct usb_serial_driver *usb_drv = to_usb_serial_driver(driver); return usb_show_dynids(&usb_drv->dynids, buf); } static DRIVER_ATTR_RW(new_id); static struct attribute *usb_serial_drv_attrs[] = { &driver_attr_new_id.attr, NULL, }; ATTRIBUTE_GROUPS(usb_serial_drv); static void free_dynids(struct usb_serial_driver *drv) { struct usb_dynid *dynid, *n; guard(mutex)(&usb_dynids_lock); list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) { list_del(&dynid->node); kfree(dynid); } } const struct bus_type usb_serial_bus_type = { .name = "usb-serial", .match = usb_serial_device_match, .probe = usb_serial_device_probe, .remove = usb_serial_device_remove, .drv_groups = usb_serial_drv_groups, }; int usb_serial_bus_register(struct usb_serial_driver *driver) { int retval; driver->driver.bus = &usb_serial_bus_type; INIT_LIST_HEAD(&driver->dynids.list); retval = driver_register(&driver->driver); return retval; } void usb_serial_bus_deregister(struct usb_serial_driver *driver) { free_dynids(driver); driver_unregister(&driver->driver); } |
| 17 17 17 17 17 17 17 16 17 17 17 16 17 17 16 17 17 17 2 4 3 3 3 2 1 2 2 2 2 2 2 4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 | // SPDX-License-Identifier: GPL-2.0-only #include <linux/bcd.h> #include <linux/delay.h> #include <linux/export.h> #include <linux/mc146818rtc.h> #ifdef CONFIG_ACPI #include <linux/acpi.h> #endif #define UIP_RECHECK_DELAY 100 /* usec */ #define UIP_RECHECK_DELAY_MS (USEC_PER_MSEC / UIP_RECHECK_DELAY) #define UIP_RECHECK_LOOPS_MS(x) (x / UIP_RECHECK_DELAY_MS) /* * Execute a function while the UIP (Update-in-progress) bit of the RTC is * unset. The timeout is configurable by the caller in ms. * * Warning: callback may be executed more then once. */ bool mc146818_avoid_UIP(void (*callback)(unsigned char seconds, void *param), int timeout, void *param) { int i; unsigned long flags; unsigned char seconds; for (i = 0; UIP_RECHECK_LOOPS_MS(i) < timeout; i++) { spin_lock_irqsave(&rtc_lock, flags); /* * Check whether there is an update in progress during which the * readout is unspecified. The maximum update time is ~2ms. Poll * for completion. * * Store the second value before checking UIP so a long lasting * NMI which happens to hit after the UIP check cannot make * an update cycle invisible. */ seconds = CMOS_READ(RTC_SECONDS); if (CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) { spin_unlock_irqrestore(&rtc_lock, flags); udelay(UIP_RECHECK_DELAY); continue; } /* Revalidate the above readout */ if (seconds != CMOS_READ(RTC_SECONDS)) { spin_unlock_irqrestore(&rtc_lock, flags); continue; } if (callback) callback(seconds, param); /* * Check for the UIP bit again. If it is set now then * the above values may contain garbage. */ if (CMOS_READ(RTC_FREQ_SELECT) & RTC_UIP) { spin_unlock_irqrestore(&rtc_lock, flags); udelay(UIP_RECHECK_DELAY); continue; } /* * A NMI might have interrupted the above sequence so check * whether the seconds value has changed which indicates that * the NMI took longer than the UIP bit was set. Unlikely, but * possible and there is also virt... */ if (seconds != CMOS_READ(RTC_SECONDS)) { spin_unlock_irqrestore(&rtc_lock, flags); continue; } spin_unlock_irqrestore(&rtc_lock, flags); if (UIP_RECHECK_LOOPS_MS(i) >= 100) pr_warn("Reading current time from RTC took around %li ms\n", UIP_RECHECK_LOOPS_MS(i)); return true; } return false; } EXPORT_SYMBOL_GPL(mc146818_avoid_UIP); /* * If the UIP (Update-in-progress) bit of the RTC is set for more then * 10ms, the RTC is apparently broken or not present. */ bool mc146818_does_rtc_work(void) { return mc146818_avoid_UIP(NULL, 1000, NULL); } EXPORT_SYMBOL_GPL(mc146818_does_rtc_work); struct mc146818_get_time_callback_param { struct rtc_time *time; unsigned char ctrl; #ifdef CONFIG_ACPI unsigned char century; #endif #ifdef CONFIG_MACH_DECSTATION unsigned int real_year; #endif }; static void mc146818_get_time_callback(unsigned char seconds, void *param_in) { struct mc146818_get_time_callback_param *p = param_in; /* * Only the values that we read from the RTC are set. We leave * tm_wday, tm_yday and tm_isdst untouched. Even though the * RTC has RTC_DAY_OF_WEEK, we ignore it, as it is only updated * by the RTC when initially set to a non-zero value. */ p->time->tm_sec = seconds; p->time->tm_min = CMOS_READ(RTC_MINUTES); p->time->tm_hour = CMOS_READ(RTC_HOURS); p->time->tm_mday = CMOS_READ(RTC_DAY_OF_MONTH); p->time->tm_mon = CMOS_READ(RTC_MONTH); p->time->tm_year = CMOS_READ(RTC_YEAR); #ifdef CONFIG_MACH_DECSTATION p->real_year = CMOS_READ(RTC_DEC_YEAR); #endif #ifdef CONFIG_ACPI if (acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID && acpi_gbl_FADT.century) { p->century = CMOS_READ(acpi_gbl_FADT.century); } else { p->century = 0; } #endif p->ctrl = CMOS_READ(RTC_CONTROL); } /** * mc146818_get_time - Get the current time from the RTC * @time: pointer to struct rtc_time to store the current time * @timeout: timeout value in ms * * This function reads the current time from the RTC and stores it in the * provided struct rtc_time. The timeout parameter specifies the maximum * time to wait for the RTC to become ready. * * Return: 0 on success, -ETIMEDOUT if the RTC did not become ready within * the specified timeout, or another error code if an error occurred. */ int mc146818_get_time(struct rtc_time *time, int timeout) { struct mc146818_get_time_callback_param p = { .time = time }; if (!mc146818_avoid_UIP(mc146818_get_time_callback, timeout, &p)) { memset(time, 0, sizeof(*time)); return -ETIMEDOUT; } if (!(p.ctrl & RTC_DM_BINARY) || RTC_ALWAYS_BCD) { time->tm_sec = bcd2bin(time->tm_sec); time->tm_min = bcd2bin(time->tm_min); time->tm_hour = bcd2bin(time->tm_hour); time->tm_mday = bcd2bin(time->tm_mday); time->tm_mon = bcd2bin(time->tm_mon); time->tm_year = bcd2bin(time->tm_year); #ifdef CONFIG_ACPI p.century = bcd2bin(p.century); #endif } #ifdef CONFIG_MACH_DECSTATION time->tm_year += p.real_year - 72; #endif #ifdef CONFIG_ACPI if (p.century > 19) time->tm_year += (p.century - 19) * 100; #endif /* * Account for differences between how the RTC uses the values * and how they are defined in a struct rtc_time; */ if (time->tm_year <= 69) time->tm_year += 100; time->tm_mon--; return 0; } EXPORT_SYMBOL_GPL(mc146818_get_time); /* AMD systems don't allow access to AltCentury with DV1 */ static bool apply_amd_register_a_behavior(void) { #ifdef CONFIG_X86 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD || boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) return true; #endif return false; } /* Set the current date and time in the real time clock. */ int mc146818_set_time(struct rtc_time *time) { unsigned long flags; unsigned char mon, day, hrs, min, sec; unsigned char save_control, save_freq_select; unsigned int yrs; #ifdef CONFIG_MACH_DECSTATION unsigned int real_yrs; #endif unsigned char century = 0; yrs = time->tm_year; mon = time->tm_mon + 1; /* tm_mon starts at zero */ day = time->tm_mday; hrs = time->tm_hour; min = time->tm_min; sec = time->tm_sec; if (yrs > 255) /* They are unsigned */ return -EINVAL; #ifdef CONFIG_MACH_DECSTATION real_yrs = yrs; yrs = 72; /* * We want to keep the year set to 73 until March * for non-leap years, so that Feb, 29th is handled * correctly. */ if (!is_leap_year(real_yrs + 1900) && mon < 3) { real_yrs--; yrs = 73; } #endif #ifdef CONFIG_ACPI if (acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID && acpi_gbl_FADT.century) { century = (yrs + 1900) / 100; yrs %= 100; } #endif /* These limits and adjustments are independent of * whether the chip is in binary mode or not. */ if (yrs > 169) return -EINVAL; if (yrs >= 100) yrs -= 100; spin_lock_irqsave(&rtc_lock, flags); save_control = CMOS_READ(RTC_CONTROL); spin_unlock_irqrestore(&rtc_lock, flags); if (!(save_control & RTC_DM_BINARY) || RTC_ALWAYS_BCD) { sec = bin2bcd(sec); min = bin2bcd(min); hrs = bin2bcd(hrs); day = bin2bcd(day); mon = bin2bcd(mon); yrs = bin2bcd(yrs); century = bin2bcd(century); } spin_lock_irqsave(&rtc_lock, flags); save_control = CMOS_READ(RTC_CONTROL); CMOS_WRITE((save_control|RTC_SET), RTC_CONTROL); save_freq_select = CMOS_READ(RTC_FREQ_SELECT); if (apply_amd_register_a_behavior()) CMOS_WRITE((save_freq_select & ~RTC_AMD_BANK_SELECT), RTC_FREQ_SELECT); else CMOS_WRITE((save_freq_select|RTC_DIV_RESET2), RTC_FREQ_SELECT); #ifdef CONFIG_MACH_DECSTATION CMOS_WRITE(real_yrs, RTC_DEC_YEAR); #endif CMOS_WRITE(yrs, RTC_YEAR); CMOS_WRITE(mon, RTC_MONTH); CMOS_WRITE(day, RTC_DAY_OF_MONTH); CMOS_WRITE(hrs, RTC_HOURS); CMOS_WRITE(min, RTC_MINUTES); CMOS_WRITE(sec, RTC_SECONDS); #ifdef CONFIG_ACPI if (acpi_gbl_FADT.header.revision >= FADT2_REVISION_ID && acpi_gbl_FADT.century) CMOS_WRITE(century, acpi_gbl_FADT.century); #endif CMOS_WRITE(save_control, RTC_CONTROL); CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT); spin_unlock_irqrestore(&rtc_lock, flags); return 0; } EXPORT_SYMBOL_GPL(mc146818_set_time); |
| 9 9 9 9 9 9 9 9 9 9 9 9 9 9 2 2 2 2 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2017, Microsoft Corporation. * Copyright (C) 2018, LG Electronics. * Copyright (c) 2025 Stefan Metzmacher */ #include "internal.h" static u8 smbdirect_ib_device_rdma_capable_node_type(struct ib_device *ib_dev) { if (!smbdirect_frwr_is_supported(&ib_dev->attrs)) return RDMA_NODE_UNSPECIFIED; switch (ib_dev->node_type) { case RDMA_NODE_IB_CA: /* Infiniband, RoCE v1 and v2 */ case RDMA_NODE_RNIC: /* iWarp */ return ib_dev->node_type; } return RDMA_NODE_UNSPECIFIED; } static int smbdirect_ib_client_add(struct ib_device *ib_dev) { u8 node_type = smbdirect_ib_device_rdma_capable_node_type(ib_dev); struct smbdirect_device *sdev; const char *node_str; const char *action; u32 pidx; switch (node_type) { case RDMA_NODE_IB_CA: node_str = "IB_CA"; action = "added"; break; case RDMA_NODE_RNIC: node_str = "RNIC"; action = "added"; break; case RDMA_NODE_UNSPECIFIED: node_str = "UNSPECIFIED"; action = "ignored"; break; default: node_str = "UNKNOWN"; action = "ignored"; node_type = RDMA_NODE_UNSPECIFIED; break; } pr_info("ib_dev[%.*s]: %s: %s %s=%u %s=0x%llx %s=0x%llx %s=0x%llx\n", IB_DEVICE_NAME_MAX, ib_dev->name, action, node_str, "max_fast_reg_page_list_len", ib_dev->attrs.max_fast_reg_page_list_len, "device_cap_flags", ib_dev->attrs.device_cap_flags, "kernel_cap_flags", ib_dev->attrs.kernel_cap_flags, "page_size_cap", ib_dev->attrs.page_size_cap); if (node_type == RDMA_NODE_UNSPECIFIED) return 0; pr_info("ib_dev[%.*s]: %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u %s=%u\n", IB_DEVICE_NAME_MAX, ib_dev->name, "num_ports", rdma_end_port(ib_dev), "max_qp_rd_atom", ib_dev->attrs.max_qp_rd_atom, "max_qp_init_rd_atom", ib_dev->attrs.max_qp_init_rd_atom, "max_sgl_rd", ib_dev->attrs.max_sgl_rd, "max_sge_rd", ib_dev->attrs.max_sge_rd, "max_cqe", ib_dev->attrs.max_cqe, "max_qp_wr", ib_dev->attrs.max_qp_wr, "max_send_sge", ib_dev->attrs.max_send_sge, "max_recv_sge", ib_dev->attrs.max_recv_sge); rdma_for_each_port(ib_dev, pidx) { const struct ib_port_immutable *ib_pi = ib_port_immutable_read(ib_dev, pidx); u32 core_cap_flags = ib_pi ? ib_pi->core_cap_flags : 0; pr_info("ib_dev[%.*s]PORT[%u]: %s=%u %s=%u %s=%u %s=%u %s=%u %s=0x%x\n", IB_DEVICE_NAME_MAX, ib_dev->name, pidx, "iwarp", rdma_protocol_iwarp(ib_dev, pidx), "ib", rdma_protocol_ib(ib_dev, pidx), "roce", rdma_protocol_roce(ib_dev, pidx), "v1", rdma_protocol_roce_eth_encap(ib_dev, pidx), "v2", rdma_protocol_roce_udp_encap(ib_dev, pidx), "core_cap_flags", core_cap_flags); } sdev = kzalloc_obj(*sdev); if (!sdev) return -ENOMEM; sdev->ib_dev = ib_dev; snprintf(sdev->ib_name, ARRAY_SIZE(sdev->ib_name), "%.*s", IB_DEVICE_NAME_MAX, ib_dev->name); write_lock(&smbdirect_globals.devices.lock); list_add(&sdev->list, &smbdirect_globals.devices.list); write_unlock(&smbdirect_globals.devices.lock); return 0; } static void smbdirect_ib_client_remove(struct ib_device *ib_dev, void *client_data) { struct smbdirect_device *sdev, *tmp; write_lock(&smbdirect_globals.devices.lock); list_for_each_entry_safe(sdev, tmp, &smbdirect_globals.devices.list, list) { if (sdev->ib_dev == ib_dev) { list_del(&sdev->list); pr_info("ib_dev[%.*s] removed\n", IB_DEVICE_NAME_MAX, sdev->ib_name); kfree(sdev); break; } } write_unlock(&smbdirect_globals.devices.lock); } static void smbdirect_ib_client_rename(struct ib_device *ib_dev, void *client_data) { struct smbdirect_device *sdev; write_lock(&smbdirect_globals.devices.lock); list_for_each_entry(sdev, &smbdirect_globals.devices.list, list) { if (sdev->ib_dev == ib_dev) { pr_info("ib_dev[%.*s] renamed to [%.*s]\n", IB_DEVICE_NAME_MAX, sdev->ib_name, IB_DEVICE_NAME_MAX, ib_dev->name); snprintf(sdev->ib_name, ARRAY_SIZE(sdev->ib_name), "%.*s", IB_DEVICE_NAME_MAX, ib_dev->name); break; } } write_unlock(&smbdirect_globals.devices.lock); } static struct ib_client smbdirect_ib_client = { .name = "smbdirect_ib_client", .add = smbdirect_ib_client_add, .remove = smbdirect_ib_client_remove, .rename = smbdirect_ib_client_rename, }; static u8 smbdirect_netdev_find_rdma_capable_node_type(struct net_device *netdev) { struct smbdirect_device *sdev; u8 node_type = RDMA_NODE_UNSPECIFIED; read_lock(&smbdirect_globals.devices.lock); list_for_each_entry(sdev, &smbdirect_globals.devices.list, list) { u32 pi; rdma_for_each_port(sdev->ib_dev, pi) { struct net_device *ndev; ndev = ib_device_get_netdev(sdev->ib_dev, pi); if (!ndev) continue; if (ndev == netdev) { dev_put(ndev); node_type = sdev->ib_dev->node_type; goto out; } dev_put(ndev); } } out: read_unlock(&smbdirect_globals.devices.lock); if (node_type == RDMA_NODE_UNSPECIFIED) { struct ib_device *ibdev; ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN); if (ibdev) { node_type = smbdirect_ib_device_rdma_capable_node_type(ibdev); ib_device_put(ibdev); } } return node_type; } /* * Returns RDMA_NODE_UNSPECIFIED when the netdev has * no support for smbdirect capable rdma. * * Otherwise RDMA_NODE_RNIC is returned for iwarp devices * and RDMA_NODE_IB_CA or Infiniband and RoCE (v1 and v2) */ u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev) { struct net_device *lower_dev; struct list_head *iter; u8 node_type = RDMA_NODE_UNSPECIFIED; node_type = smbdirect_netdev_find_rdma_capable_node_type(netdev); if (node_type != RDMA_NODE_UNSPECIFIED) return node_type; /* check if netdev is bridge or VLAN */ if (netif_is_bridge_master(netdev) || netdev->priv_flags & IFF_802_1Q_VLAN) netdev_for_each_lower_dev(netdev, lower_dev, iter) { node_type = smbdirect_netdev_find_rdma_capable_node_type(lower_dev); if (node_type != RDMA_NODE_UNSPECIFIED) return node_type; } /* check if netdev is IPoIB safely without layer violation */ if (netdev->type == ARPHRD_INFINIBAND) return RDMA_NODE_IB_CA; return RDMA_NODE_UNSPECIFIED; } EXPORT_SYMBOL_GPL(smbdirect_netdev_rdma_capable_node_type); __init int smbdirect_devices_init(void) { int ret; rwlock_init(&smbdirect_globals.devices.lock); INIT_LIST_HEAD(&smbdirect_globals.devices.list); ret = ib_register_client(&smbdirect_ib_client); if (ret) { pr_crit("failed to ib_register_client: %d %1pe\n", ret, SMBDIRECT_DEBUG_ERR_PTR(ret)); return ret; } return 0; } __exit void smbdirect_devices_exit(void) { struct smbdirect_device *sdev, *tmp; /* * On exist we just cleanup so that * smbdirect_ib_client_remove() won't * print removals of devices. */ write_lock(&smbdirect_globals.devices.lock); list_for_each_entry_safe(sdev, tmp, &smbdirect_globals.devices.list, list) { list_del(&sdev->list); kfree(sdev); } write_unlock(&smbdirect_globals.devices.lock); ib_unregister_client(&smbdirect_ib_client); } |
| 3296 148 1 148 234 2 2 449 447 239 237 232 444 448 449 151 237 447 2 229 126 189 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (c) 2025 Oracle. All Rights Reserved. * Author: Darrick J. Wong <djwong@kernel.org> */ #include <linux/fs.h> #include <linux/fsnotify.h> #include <linux/mempool.h> #include <linux/fserror.h> #define FSERROR_DEFAULT_EVENT_POOL_SIZE (32) static struct mempool fserror_events_pool; void fserror_mount(struct super_block *sb) { /* * The pending error counter is biased by 1 so that we don't wake_var * until we're actually trying to unmount. */ refcount_set(&sb->s_pending_errors, 1); } void fserror_unmount(struct super_block *sb) { /* * If we don't drop the pending error count to zero, then wait for it * to drop below 1, which means that the pending errors cleared and * hopefully we didn't saturate with 1 billion+ concurrent events. */ if (!refcount_dec_and_test(&sb->s_pending_errors)) wait_var_event(&sb->s_pending_errors, refcount_read(&sb->s_pending_errors) < 1); } static inline void fserror_pending_dec(struct super_block *sb) { if (refcount_dec_and_test(&sb->s_pending_errors)) wake_up_var(&sb->s_pending_errors); } static inline void fserror_free_event(struct fserror_event *event) { fserror_pending_dec(event->sb); mempool_free(event, &fserror_events_pool); } static void fserror_worker(struct work_struct *work) { struct fserror_event *event = container_of(work, struct fserror_event, work); struct super_block *sb = event->sb; if (sb->s_flags & SB_ACTIVE) { struct fs_error_report report = { /* send positive error number to userspace */ .error = -event->error, .inode = event->inode, .sb = event->sb, }; if (sb->s_op->report_error) sb->s_op->report_error(event); fsnotify(FS_ERROR, &report, FSNOTIFY_EVENT_ERROR, NULL, NULL, NULL, 0); } iput(event->inode); fserror_free_event(event); } static inline struct fserror_event *fserror_alloc_event(struct super_block *sb, gfp_t gfp_flags) { struct fserror_event *event = NULL; /* * If pending_errors already reached zero or is no longer active, * the superblock is being deactivated so there's no point in * continuing. * * The order of the check of s_pending_errors and SB_ACTIVE are * mandated by order of accesses in generic_shutdown_super and * fserror_unmount. Barriers are implicitly provided by the refcount * manipulations in this function and fserror_unmount. */ if (!refcount_inc_not_zero(&sb->s_pending_errors)) return NULL; if (!(sb->s_flags & SB_ACTIVE)) goto out_pending; event = mempool_alloc(&fserror_events_pool, gfp_flags); if (!event) goto out_pending; /* mempool_alloc doesn't support GFP_ZERO */ memset(event, 0, sizeof(*event)); event->sb = sb; INIT_WORK(&event->work, fserror_worker); return event; out_pending: fserror_pending_dec(sb); return NULL; } /** * fserror_report - report a filesystem error of some kind * * @sb: superblock of the filesystem * @inode: inode within that filesystem, if applicable * @type: type of error encountered * @pos: start of inode range affected, if applicable * @len: length of inode range affected, if applicable * @error: error number encountered, must be negative * @gfp: memory allocation flags for conveying the event to a worker, * since this function can be called from atomic contexts * * Report details of a filesystem error to the super_operations::report_error * callback if present; and to fsnotify for distribution to userspace. @sb, * @gfp, @type, and @error must all be specified. For file I/O errors, the * @inode, @pos, and @len fields must also be specified. For file metadata * errors, @inode must be specified. If @inode is not NULL, then @inode->i_sb * must point to @sb. * * Reporting work is deferred to a workqueue to ensure that ->report_error is * called from process context without any locks held. An active reference to * the inode is maintained until event handling is complete, and unmount will * wait for queued events to drain. */ void fserror_report(struct super_block *sb, struct inode *inode, enum fserror_type type, loff_t pos, u64 len, int error, gfp_t gfp) { struct fserror_event *event; /* sb and inode must be from the same filesystem */ WARN_ON_ONCE(inode && inode->i_sb != sb); /* error number must be negative */ WARN_ON_ONCE(error >= 0); event = fserror_alloc_event(sb, gfp); if (!event) goto lost; event->type = type; event->pos = pos; event->len = len; event->error = error; /* * Can't iput from non-sleeping context, so grabbing another reference * to the inode must be the last thing before submitting the event. */ if (inode) { event->inode = igrab(inode); if (!event->inode) goto lost_event; } /* * Use schedule_work here even if we're already in process context so * that fsnotify and super_operations::report_error implementations are * guaranteed to run in process context without any locks held. Since * errors are supposed to be rare, the overhead shouldn't kill us any * more than the failing device will. */ schedule_work(&event->work); return; lost_event: fserror_free_event(event); lost: if (inode) pr_err_ratelimited( "%s: lost file I/O error report for ino %llu type %u pos 0x%llx len 0x%llx error %d", sb->s_id, inode->i_ino, type, pos, len, error); else pr_err_ratelimited( "%s: lost filesystem error report for type %u error %d", sb->s_id, type, error); } EXPORT_SYMBOL_GPL(fserror_report); static int __init fserror_init(void) { return mempool_init_kmalloc_pool(&fserror_events_pool, FSERROR_DEFAULT_EVENT_POOL_SIZE, sizeof(struct fserror_event)); } fs_initcall(fserror_init); |
| 2 2 2 2 1 2 1 1 1 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | // SPDX-License-Identifier: GPL-2.0-only /* * V9FS cache definitions. * * Copyright (C) 2009 by Abhishek Kulkarni <adkulkar@umail.iu.edu> */ #include <linux/jiffies.h> #include <linux/file.h> #include <linux/slab.h> #include <linux/stat.h> #include <linux/sched.h> #include <linux/fs.h> #include <net/9p/9p.h> #include "v9fs.h" #include "cache.h" int v9fs_cache_session_get_cookie(struct v9fs_session_info *v9ses, const char *dev_name) { struct fscache_volume *vcookie; char *name, *p; name = kasprintf(GFP_KERNEL, "9p,%s,%s", dev_name, v9ses->cachetag ?: v9ses->aname); if (!name) return -ENOMEM; for (p = name; *p; p++) if (*p == '/') *p = ';'; vcookie = fscache_acquire_volume(name, NULL, NULL, 0); p9_debug(P9_DEBUG_FSC, "session %p get volume %p (%s)\n", v9ses, vcookie, name); if (IS_ERR(vcookie)) { if (vcookie != ERR_PTR(-EBUSY)) { kfree(name); return PTR_ERR(vcookie); } pr_err("Cache volume key already in use (%s)\n", name); vcookie = NULL; } v9ses->fscache = vcookie; kfree(name); return 0; } void v9fs_cache_inode_get_cookie(struct inode *inode) { struct v9fs_inode *v9inode = V9FS_I(inode); struct v9fs_session_info *v9ses; __le32 version; __le64 path; if (!S_ISREG(inode->i_mode)) return; if (WARN_ON(v9fs_inode_cookie(v9inode))) return; version = cpu_to_le32(v9inode->qid.version); path = cpu_to_le64(v9inode->qid.path); v9ses = v9fs_inode2v9ses(inode); v9inode->netfs.cache = fscache_acquire_cookie(v9fs_session_cache(v9ses), 0, &path, sizeof(path), &version, sizeof(version), i_size_read(&v9inode->netfs.inode)); if (v9inode->netfs.cache) mapping_set_release_always(inode->i_mapping); p9_debug(P9_DEBUG_FSC, "inode %p get cookie %p\n", inode, v9fs_inode_cookie(v9inode)); } |
| 2 2 2 2 2 2 2 2 2 2 1 1 2 2 2 2 1 2 1 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 93 93 89 8 8 8 1 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 | // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) 2011, 2012 STRATO. All rights reserved. */ #include <linux/blkdev.h> #include <linux/ratelimit.h> #include <linux/sched/mm.h> #include "ctree.h" #include "discard.h" #include "volumes.h" #include "disk-io.h" #include "ordered-data.h" #include "transaction.h" #include "backref.h" #include "extent_io.h" #include "dev-replace.h" #include "raid56.h" #include "block-group.h" #include "zoned.h" #include "fs.h" #include "accessors.h" #include "file-item.h" #include "scrub.h" #include "raid-stripe-tree.h" /* * This is only the first step towards a full-features scrub. It reads all * extent and super block and verifies the checksums. In case a bad checksum * is found or the extent cannot be read, good data will be written back if * any can be found. * * Future enhancements: * - In case an unrepairable extent is encountered, track which files are * affected and report them * - track and record media errors, throw out bad devices * - add a mode to also read unallocated space */ struct scrub_ctx; /* * The following value only influences the performance. * * This determines how many stripes would be submitted in one go, * which is 512KiB (BTRFS_STRIPE_LEN * SCRUB_STRIPES_PER_GROUP). */ #define SCRUB_STRIPES_PER_GROUP 8 /* * How many groups we have for each sctx. * * This would be 8M per device, the same value as the old scrub in-flight bios * size limit. */ #define SCRUB_GROUPS_PER_SCTX 16 #define SCRUB_TOTAL_STRIPES (SCRUB_GROUPS_PER_SCTX * SCRUB_STRIPES_PER_GROUP) /* * The following value times PAGE_SIZE needs to be large enough to match the * largest node/leaf/sector size that shall be supported. */ #define SCRUB_MAX_SECTORS_PER_BLOCK (BTRFS_MAX_METADATA_BLOCKSIZE / SZ_4K) /* Represent one sector and its needed info to verify the content. */ struct scrub_sector_verification { union { /* * Csum pointer for data csum verification. Should point to a * sector csum inside scrub_stripe::csums. * * NULL if this data sector has no csum. */ u8 *csum; /* * Extra info for metadata verification. All sectors inside a * tree block share the same generation. */ u64 generation; }; }; enum scrub_stripe_flags { /* Set when @mirror_num, @dev, @physical and @logical are set. */ SCRUB_STRIPE_FLAG_INITIALIZED, /* Set when the read-repair is finished. */ SCRUB_STRIPE_FLAG_REPAIR_DONE, /* * Set for data stripes if it's triggered from P/Q stripe. * During such scrub, we should not report errors in data stripes, nor * update the accounting. */ SCRUB_STRIPE_FLAG_NO_REPORT, }; /* * We have multiple bitmaps for one scrub_stripe. * However each bitmap has at most (BTRFS_STRIPE_LEN / blocksize) bits, * which is normally 16, and much smaller than BITS_PER_LONG (32 or 64). * * So to reduce memory usage for each scrub_stripe, we pack those bitmaps * into a larger one. * * These enum records where the sub-bitmap are inside the larger one. * Each subbitmap starts at scrub_bitmap_nr_##name * nr_sectors bit. */ enum { /* Which blocks are covered by extent items. */ scrub_bitmap_nr_has_extent = 0, /* Which blocks are metadata. */ scrub_bitmap_nr_is_metadata, /* * Which blocks have errors, including IO, csum, and metadata * errors. * This sub-bitmap is the OR results of the next few error related * sub-bitmaps. */ scrub_bitmap_nr_error, scrub_bitmap_nr_io_error, scrub_bitmap_nr_csum_error, scrub_bitmap_nr_meta_error, scrub_bitmap_nr_meta_gen_error, scrub_bitmap_nr_last, }; #define SCRUB_STRIPE_MAX_FOLIOS (BTRFS_STRIPE_LEN / PAGE_SIZE) /* * Represent one contiguous range with a length of BTRFS_STRIPE_LEN. */ struct scrub_stripe { struct scrub_ctx *sctx; struct btrfs_block_group *bg; struct folio *folios[SCRUB_STRIPE_MAX_FOLIOS]; struct scrub_sector_verification *sectors; struct btrfs_device *dev; u64 logical; u64 physical; u16 mirror_num; /* Should be BTRFS_STRIPE_LEN / sectorsize. */ u16 nr_sectors; /* * How many data/meta extents are in this stripe. Only for scrub status * reporting purposes. */ u16 nr_data_extents; u16 nr_meta_extents; atomic_t pending_io; wait_queue_head_t io_wait; wait_queue_head_t repair_wait; /* * Indicate the states of the stripe. Bits are defined in * scrub_stripe_flags enum. */ unsigned long state; /* The large bitmap contains all the sub-bitmaps. */ unsigned long bitmaps[BITS_TO_LONGS(scrub_bitmap_nr_last * (BTRFS_STRIPE_LEN / BTRFS_MIN_BLOCKSIZE))]; /* * For writeback (repair or replace) error reporting. * This one is protected by a spinlock, thus can not be packed into * the larger bitmap. */ unsigned long write_error_bitmap; /* Writeback can be concurrent, thus we need to protect the bitmap. */ spinlock_t write_error_lock; /* * Checksum for the whole stripe if this stripe is inside a data block * group. */ u8 *csums; struct work_struct work; }; struct scrub_ctx { struct scrub_stripe stripes[SCRUB_TOTAL_STRIPES]; struct scrub_stripe *raid56_data_stripes; struct btrfs_fs_info *fs_info; struct btrfs_path extent_path; struct btrfs_path csum_path; int first_free; int cur_stripe; atomic_t cancel_req; int readonly; /* State of IO submission throttling affecting the associated device */ ktime_t throttle_deadline; u64 throttle_sent; bool is_dev_replace; u64 write_pointer; struct mutex wr_lock; struct btrfs_device *wr_tgtdev; /* * statistics */ struct btrfs_scrub_progress stat; spinlock_t stat_lock; /* * Use a ref counter to avoid use-after-free issues. Scrub workers * decrement bios_in_flight and workers_pending and then do a wakeup * on the list_wait wait queue. We must ensure the main scrub task * doesn't free the scrub context before or while the workers are * doing the wakeup() call. */ refcount_t refs; }; #define scrub_calc_start_bit(stripe, name, block_nr) \ ({ \ unsigned int __start_bit; \ \ ASSERT(block_nr < stripe->nr_sectors, \ "nr_sectors=%u block_nr=%u", stripe->nr_sectors, block_nr); \ __start_bit = scrub_bitmap_nr_##name * stripe->nr_sectors + block_nr; \ __start_bit; \ }) #define IMPLEMENT_SCRUB_BITMAP_OPS(name) \ static inline void scrub_bitmap_set_##name(struct scrub_stripe *stripe, \ unsigned int block_nr, \ unsigned int nr_blocks) \ { \ const unsigned int start_bit = scrub_calc_start_bit(stripe, \ name, block_nr); \ \ bitmap_set(stripe->bitmaps, start_bit, nr_blocks); \ } \ static inline void scrub_bitmap_clear_##name(struct scrub_stripe *stripe, \ unsigned int block_nr, \ unsigned int nr_blocks) \ { \ const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \ block_nr); \ \ bitmap_clear(stripe->bitmaps, start_bit, nr_blocks); \ } \ static inline bool scrub_bitmap_test_bit_##name(struct scrub_stripe *stripe, \ unsigned int block_nr) \ { \ const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \ block_nr); \ \ return test_bit(start_bit, stripe->bitmaps); \ } \ static inline void scrub_bitmap_set_bit_##name(struct scrub_stripe *stripe, \ unsigned int block_nr) \ { \ const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \ block_nr); \ \ set_bit(start_bit, stripe->bitmaps); \ } \ static inline void scrub_bitmap_clear_bit_##name(struct scrub_stripe *stripe, \ unsigned int block_nr) \ { \ const unsigned int start_bit = scrub_calc_start_bit(stripe, name, \ block_nr); \ \ clear_bit(start_bit, stripe->bitmaps); \ } \ static inline unsigned long scrub_bitmap_read_##name(struct scrub_stripe *stripe) \ { \ const unsigned int nr_blocks = stripe->nr_sectors; \ \ ASSERT(nr_blocks > 0 && nr_blocks <= BITS_PER_LONG, \ "nr_blocks=%u BITS_PER_LONG=%u", \ nr_blocks, BITS_PER_LONG); \ \ return bitmap_read(stripe->bitmaps, nr_blocks * scrub_bitmap_nr_##name, \ stripe->nr_sectors); \ } \ static inline bool scrub_bitmap_empty_##name(struct scrub_stripe *stripe) \ { \ unsigned long bitmap = scrub_bitmap_read_##name(stripe); \ \ return bitmap_empty(&bitmap, stripe->nr_sectors); \ } \ static inline unsigned int scrub_bitmap_weight_##name(struct scrub_stripe *stripe) \ { \ unsigned long bitmap = scrub_bitmap_read_##name(stripe); \ \ return bitmap_weight(&bitmap, stripe->nr_sectors); \ } IMPLEMENT_SCRUB_BITMAP_OPS(has_extent); IMPLEMENT_SCRUB_BITMAP_OPS(is_metadata); IMPLEMENT_SCRUB_BITMAP_OPS(error); IMPLEMENT_SCRUB_BITMAP_OPS(io_error); IMPLEMENT_SCRUB_BITMAP_OPS(csum_error); IMPLEMENT_SCRUB_BITMAP_OPS(meta_error); IMPLEMENT_SCRUB_BITMAP_OPS(meta_gen_error); struct scrub_warning { struct btrfs_path *path; u64 extent_item_size; const char *errstr; u64 physical; u64 logical; struct btrfs_device *dev; }; struct scrub_error_records { /* * Bitmap recording which blocks hit errors (IO/csum/...) during the * initial read. */ unsigned long init_error_bitmap; unsigned int nr_io_errors; unsigned int nr_csum_errors; unsigned int nr_meta_errors; unsigned int nr_meta_gen_errors; }; static void release_scrub_stripe(struct scrub_stripe *stripe) { if (!stripe) return; for (int i = 0; i < SCRUB_STRIPE_MAX_FOLIOS; i++) { if (stripe->folios[i]) folio_put(stripe->folios[i]); stripe->folios[i] = NULL; } kfree(stripe->sectors); kfree(stripe->csums); stripe->sectors = NULL; stripe->csums = NULL; stripe->sctx = NULL; stripe->state = 0; } static int init_scrub_stripe(struct btrfs_fs_info *fs_info, struct scrub_stripe *stripe) { const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order; int ret; memset(stripe, 0, sizeof(*stripe)); stripe->nr_sectors = BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits; stripe->state = 0; init_waitqueue_head(&stripe->io_wait); init_waitqueue_head(&stripe->repair_wait); atomic_set(&stripe->pending_io, 0); spin_lock_init(&stripe->write_error_lock); ASSERT(BTRFS_STRIPE_LEN >> min_folio_shift <= SCRUB_STRIPE_MAX_FOLIOS); ret = btrfs_alloc_folio_array(BTRFS_STRIPE_LEN >> min_folio_shift, fs_info->block_min_order, stripe->folios, GFP_NOFS); if (ret < 0) goto error; stripe->sectors = kzalloc_objs(struct scrub_sector_verification, stripe->nr_sectors); if (!stripe->sectors) goto error; stripe->csums = kcalloc(BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits, fs_info->csum_size, GFP_KERNEL); if (!stripe->csums) goto error; return 0; error: release_scrub_stripe(stripe); return -ENOMEM; } static void wait_scrub_stripe_io(struct scrub_stripe *stripe) { wait_event(stripe->io_wait, atomic_read(&stripe->pending_io) == 0); } static void scrub_put_ctx(struct scrub_ctx *sctx); static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info) { while (atomic_read(&fs_info->scrub_pause_req)) { mutex_unlock(&fs_info->scrub_lock); wait_event(fs_info->scrub_pause_wait, atomic_read(&fs_info->scrub_pause_req) == 0); mutex_lock(&fs_info->scrub_lock); } } static void scrub_pause_on(struct btrfs_fs_info *fs_info) { atomic_inc(&fs_info->scrubs_paused); wake_up(&fs_info->scrub_pause_wait); } static void scrub_pause_off(struct btrfs_fs_info *fs_info) { mutex_lock(&fs_info->scrub_lock); __scrub_blocked_if_needed(fs_info); atomic_dec(&fs_info->scrubs_paused); mutex_unlock(&fs_info->scrub_lock); wake_up(&fs_info->scrub_pause_wait); } static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info) { scrub_pause_on(fs_info); scrub_pause_off(fs_info); } static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx) { int i; if (!sctx) return; for (i = 0; i < SCRUB_TOTAL_STRIPES; i++) release_scrub_stripe(&sctx->stripes[i]); kvfree(sctx); } static void scrub_put_ctx(struct scrub_ctx *sctx) { if (refcount_dec_and_test(&sctx->refs)) scrub_free_ctx(sctx); } static noinline_for_stack struct scrub_ctx *scrub_setup_ctx( struct btrfs_fs_info *fs_info, bool is_dev_replace) { struct scrub_ctx *sctx; int i; /* Since sctx has inline 128 stripes, it can go beyond 64K easily. Use * kvzalloc(). */ sctx = kvzalloc_obj(*sctx); if (!sctx) goto nomem; refcount_set(&sctx->refs, 1); sctx->is_dev_replace = is_dev_replace; sctx->fs_info = fs_info; sctx->extent_path.search_commit_root = true; sctx->extent_path.skip_locking = true; sctx->csum_path.search_commit_root = true; sctx->csum_path.skip_locking = true; for (i = 0; i < SCRUB_TOTAL_STRIPES; i++) { int ret; ret = init_scrub_stripe(fs_info, &sctx->stripes[i]); if (ret < 0) goto nomem; sctx->stripes[i].sctx = sctx; } sctx->first_free = 0; atomic_set(&sctx->cancel_req, 0); spin_lock_init(&sctx->stat_lock); sctx->throttle_deadline = 0; mutex_init(&sctx->wr_lock); if (is_dev_replace) { WARN_ON(!fs_info->dev_replace.tgtdev); sctx->wr_tgtdev = fs_info->dev_replace.tgtdev; } return sctx; nomem: scrub_free_ctx(sctx); return ERR_PTR(-ENOMEM); } static int scrub_print_warning_inode(u64 inum, u64 offset, u64 num_bytes, u64 root, void *warn_ctx) { u32 nlink; int ret; int i; unsigned nofs_flag; struct extent_buffer *eb; struct btrfs_inode_item *inode_item; struct scrub_warning *swarn = warn_ctx; struct btrfs_fs_info *fs_info = swarn->dev->fs_info; struct inode_fs_paths *ipath __free(inode_fs_paths) = NULL; struct btrfs_root *local_root; struct btrfs_key key; local_root = btrfs_get_fs_root(fs_info, root, true); if (IS_ERR(local_root)) { ret = PTR_ERR(local_root); goto err; } /* * this makes the path point to (inum INODE_ITEM ioff) */ key.objectid = inum; key.type = BTRFS_INODE_ITEM_KEY; key.offset = 0; ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0); if (ret) { btrfs_put_root(local_root); btrfs_release_path(swarn->path); goto err; } eb = swarn->path->nodes[0]; inode_item = btrfs_item_ptr(eb, swarn->path->slots[0], struct btrfs_inode_item); nlink = btrfs_inode_nlink(eb, inode_item); btrfs_release_path(swarn->path); /* * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub * uses GFP_NOFS in this context, so we keep it consistent but it does * not seem to be strictly necessary. */ nofs_flag = memalloc_nofs_save(); ipath = init_ipath(4096, local_root, swarn->path); memalloc_nofs_restore(nofs_flag); if (IS_ERR(ipath)) { btrfs_put_root(local_root); ret = PTR_ERR(ipath); ipath = NULL; goto err; } ret = paths_from_inode(inum, ipath); if (ret < 0) goto err; /* * we deliberately ignore the bit ipath might have been too small to * hold all of the paths here */ for (i = 0; i < ipath->fspath->elem_cnt; ++i) btrfs_warn(fs_info, "scrub: %s at logical %llu on dev %s, physical %llu root %llu inode %llu offset %llu length %u links %u (path: %s)", swarn->errstr, swarn->logical, btrfs_dev_name(swarn->dev), swarn->physical, root, inum, offset, fs_info->sectorsize, nlink, (char *)(unsigned long)ipath->fspath->val[i]); btrfs_put_root(local_root); return 0; err: btrfs_warn(fs_info, "scrub: %s at logical %llu on dev %s, physical %llu root %llu inode %llu offset %llu: path resolving failed with ret=%d", swarn->errstr, swarn->logical, btrfs_dev_name(swarn->dev), swarn->physical, root, inum, offset, ret); return 0; } static void scrub_print_common_warning(const char *errstr, struct btrfs_device *dev, bool is_super, u64 logical, u64 physical) { struct btrfs_fs_info *fs_info = dev->fs_info; BTRFS_PATH_AUTO_FREE(path); struct btrfs_key found_key; struct extent_buffer *eb; struct btrfs_extent_item *ei; struct scrub_warning swarn; u64 flags = 0; u32 item_size; int ret; /* Super block error, no need to search extent tree. */ if (is_super) { btrfs_warn(fs_info, "scrub: %s on device %s, physical %llu", errstr, btrfs_dev_name(dev), physical); return; } path = btrfs_alloc_path(); if (!path) return; swarn.physical = physical; swarn.logical = logical; swarn.errstr = errstr; swarn.dev = NULL; ret = extent_from_logical(fs_info, swarn.logical, path, &found_key, &flags); if (ret < 0) return; swarn.extent_item_size = found_key.offset; eb = path->nodes[0]; ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item); item_size = btrfs_item_size(eb, path->slots[0]); if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { unsigned long ptr = 0; u8 ref_level; u64 ref_root; while (true) { ret = tree_backref_for_extent(&ptr, eb, &found_key, ei, item_size, &ref_root, &ref_level); if (ret < 0) { btrfs_warn(fs_info, "scrub: failed to resolve tree backref for logical %llu: %d", swarn.logical, ret); break; } if (ret > 0) break; btrfs_warn(fs_info, "scrub: %s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu", errstr, swarn.logical, btrfs_dev_name(dev), swarn.physical, (ref_level ? "node" : "leaf"), ref_level, ref_root); } btrfs_release_path(path); } else { struct btrfs_backref_walk_ctx ctx = { 0 }; btrfs_release_path(path); ctx.bytenr = found_key.objectid; ctx.extent_item_pos = swarn.logical - found_key.objectid; ctx.fs_info = fs_info; swarn.path = path; swarn.dev = dev; iterate_extent_inodes(&ctx, true, scrub_print_warning_inode, &swarn); } } static int fill_writer_pointer_gap(struct scrub_ctx *sctx, u64 physical) { int ret = 0; u64 length; if (!btrfs_is_zoned(sctx->fs_info)) return 0; if (!btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) return 0; if (sctx->write_pointer < physical) { length = physical - sctx->write_pointer; ret = btrfs_zoned_issue_zeroout(sctx->wr_tgtdev, sctx->write_pointer, length); if (!ret) sctx->write_pointer = physical; } return ret; } static void *scrub_stripe_get_kaddr(struct scrub_stripe *stripe, int sector_nr) { struct btrfs_fs_info *fs_info = stripe->bg->fs_info; const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order; u32 offset = (sector_nr << fs_info->sectorsize_bits); const struct folio *folio = stripe->folios[offset >> min_folio_shift]; /* stripe->folios[] is allocated by us and no highmem is allowed. */ ASSERT(folio); ASSERT(!folio_test_highmem(folio)); return folio_address(folio) + offset_in_folio(folio, offset); } static phys_addr_t scrub_stripe_get_paddr(struct scrub_stripe *stripe, int sector_nr) { struct btrfs_fs_info *fs_info = stripe->bg->fs_info; const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order; u32 offset = (sector_nr << fs_info->sectorsize_bits); const struct folio *folio = stripe->folios[offset >> min_folio_shift]; /* stripe->folios[] is allocated by us and no highmem is allowed. */ ASSERT(folio); ASSERT(!folio_test_highmem(folio)); /* And the range must be contained inside the folio. */ ASSERT(offset_in_folio(folio, offset) + fs_info->sectorsize <= folio_size(folio)); return page_to_phys(folio_page(folio, 0)) + offset_in_folio(folio, offset); } static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr) { struct btrfs_fs_info *fs_info = stripe->bg->fs_info; const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits; const u64 logical = stripe->logical + (sector_nr << fs_info->sectorsize_bits); void *first_kaddr = scrub_stripe_get_kaddr(stripe, sector_nr); struct btrfs_header *header = first_kaddr; struct btrfs_csum_ctx csum; u8 on_disk_csum[BTRFS_CSUM_SIZE]; u8 calculated_csum[BTRFS_CSUM_SIZE]; /* * Here we don't have a good way to attach the pages (and subpages) * to a dummy extent buffer, thus we have to directly grab the members * from pages. */ memcpy(on_disk_csum, header->csum, fs_info->csum_size); if (logical != btrfs_stack_header_bytenr(header)) { scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree); scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); btrfs_warn_rl(fs_info, "scrub: tree block %llu mirror %u has bad bytenr, has %llu want %llu", logical, stripe->mirror_num, btrfs_stack_header_bytenr(header), logical); return; } if (memcmp(header->fsid, fs_info->fs_devices->metadata_uuid, BTRFS_FSID_SIZE) != 0) { scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree); scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); btrfs_warn_rl(fs_info, "scrub: tree block %llu mirror %u has bad fsid, has %pU want %pU", logical, stripe->mirror_num, header->fsid, fs_info->fs_devices->metadata_uuid); return; } if (memcmp(header->chunk_tree_uuid, fs_info->chunk_tree_uuid, BTRFS_UUID_SIZE) != 0) { scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree); scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); btrfs_warn_rl(fs_info, "scrub: tree block %llu mirror %u has bad chunk tree uuid, has %pU want %pU", logical, stripe->mirror_num, header->chunk_tree_uuid, fs_info->chunk_tree_uuid); return; } /* Now check tree block csum. */ btrfs_csum_init(&csum, fs_info->csum_type); btrfs_csum_update(&csum, first_kaddr + BTRFS_CSUM_SIZE, fs_info->sectorsize - BTRFS_CSUM_SIZE); for (int i = sector_nr + 1; i < sector_nr + sectors_per_tree; i++) { btrfs_csum_update(&csum, scrub_stripe_get_kaddr(stripe, i), fs_info->sectorsize); } btrfs_csum_final(&csum, calculated_csum); if (memcmp(calculated_csum, on_disk_csum, fs_info->csum_size) != 0) { scrub_bitmap_set_meta_error(stripe, sector_nr, sectors_per_tree); scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); btrfs_warn_rl(fs_info, "scrub: tree block %llu mirror %u has bad csum, has " BTRFS_CSUM_FMT " want " BTRFS_CSUM_FMT, logical, stripe->mirror_num, BTRFS_CSUM_FMT_VALUE(fs_info->csum_size, on_disk_csum), BTRFS_CSUM_FMT_VALUE(fs_info->csum_size, calculated_csum)); return; } if (stripe->sectors[sector_nr].generation != btrfs_stack_header_generation(header)) { scrub_bitmap_set_meta_gen_error(stripe, sector_nr, sectors_per_tree); scrub_bitmap_set_error(stripe, sector_nr, sectors_per_tree); btrfs_warn_rl(fs_info, "scrub: tree block %llu mirror %u has bad generation, has %llu want %llu", logical, stripe->mirror_num, btrfs_stack_header_generation(header), stripe->sectors[sector_nr].generation); return; } scrub_bitmap_clear_error(stripe, sector_nr, sectors_per_tree); scrub_bitmap_clear_csum_error(stripe, sector_nr, sectors_per_tree); scrub_bitmap_clear_meta_error(stripe, sector_nr, sectors_per_tree); scrub_bitmap_clear_meta_gen_error(stripe, sector_nr, sectors_per_tree); } static void scrub_verify_one_sector(struct scrub_stripe *stripe, int sector_nr) { struct btrfs_fs_info *fs_info = stripe->bg->fs_info; struct scrub_sector_verification *sector = &stripe->sectors[sector_nr]; const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits; phys_addr_t paddr = scrub_stripe_get_paddr(stripe, sector_nr); u8 csum_buf[BTRFS_CSUM_SIZE]; int ret; ASSERT(sector_nr >= 0 && sector_nr < stripe->nr_sectors); /* Sector not utilized, skip it. */ if (!scrub_bitmap_test_bit_has_extent(stripe, sector_nr)) return; /* IO error, no need to check. */ if (scrub_bitmap_test_bit_io_error(stripe, sector_nr)) return; /* Metadata, verify the full tree block. */ if (scrub_bitmap_test_bit_is_metadata(stripe, sector_nr)) { /* * Check if the tree block crosses the stripe boundary. If * crossed the boundary, we cannot verify it but only give a * warning. * * This can only happen on a very old filesystem where chunks * are not ensured to be stripe aligned. */ if (unlikely(sector_nr + sectors_per_tree > stripe->nr_sectors)) { btrfs_warn_rl(fs_info, "scrub: tree block at %llu crosses stripe boundary %llu", stripe->logical + (sector_nr << fs_info->sectorsize_bits), stripe->logical); return; } scrub_verify_one_metadata(stripe, sector_nr); return; } /* * Data is easier, we just verify the data csum (if we have it). For * cases without csum, we have no other choice but to trust it. */ if (!sector->csum) { scrub_bitmap_clear_bit_error(stripe, sector_nr); return; } ret = btrfs_check_block_csum(fs_info, paddr, csum_buf, sector->csum); if (ret < 0) { scrub_bitmap_set_bit_csum_error(stripe, sector_nr); scrub_bitmap_set_bit_error(stripe, sector_nr); } else { scrub_bitmap_clear_bit_csum_error(stripe, sector_nr); scrub_bitmap_clear_bit_error(stripe, sector_nr); } } /* Verify specified sectors of a stripe. */ static void scrub_verify_one_stripe(struct scrub_stripe *stripe, unsigned long bitmap) { struct btrfs_fs_info *fs_info = stripe->bg->fs_info; const u32 sectors_per_tree = fs_info->nodesize >> fs_info->sectorsize_bits; int sector_nr; for_each_set_bit(sector_nr, &bitmap, stripe->nr_sectors) { scrub_verify_one_sector(stripe, sector_nr); if (scrub_bitmap_test_bit_is_metadata(stripe, sector_nr)) sector_nr += sectors_per_tree - 1; } } static int calc_sector_number(struct scrub_stripe *stripe, struct bio_vec *first_bvec) { int i; for (i = 0; i < stripe->nr_sectors; i++) { if (scrub_stripe_get_kaddr(stripe, i) == bvec_virt(first_bvec)) break; } ASSERT(i < stripe->nr_sectors); return i; } /* * Repair read is different to the regular read: * * - Only reads the failed sectors * - May have extra blocksize limits */ static void scrub_repair_read_endio(struct btrfs_bio *bbio) { struct scrub_stripe *stripe = bbio->private; struct btrfs_fs_info *fs_info = stripe->bg->fs_info; int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio)); const u32 bio_size = bio_get_size(&bbio->bio); ASSERT(sector_nr < stripe->nr_sectors); if (bbio->bio.bi_status) { scrub_bitmap_set_io_error(stripe, sector_nr, bio_size >> fs_info->sectorsize_bits); scrub_bitmap_set_error(stripe, sector_nr, bio_size >> fs_info->sectorsize_bits); } else { scrub_bitmap_clear_io_error(stripe, sector_nr, bio_size >> fs_info->sectorsize_bits); } bio_put(&bbio->bio); if (atomic_dec_and_test(&stripe->pending_io)) wake_up(&stripe->io_wait); } static int calc_next_mirror(int mirror, int num_copies) { ASSERT(mirror <= num_copies); return (mirror + 1 > num_copies) ? 1 : mirror + 1; } static void scrub_bio_add_sector(struct btrfs_bio *bbio, struct scrub_stripe *stripe, int sector_nr) { struct btrfs_fs_info *fs_info = bbio->inode->root->fs_info; void *kaddr = scrub_stripe_get_kaddr(stripe, sector_nr); int ret; ret = bio_add_page(&bbio->bio, virt_to_page(kaddr), fs_info->sectorsize, offset_in_page(kaddr)); /* * Caller should ensure the bbio has enough size. * And we cannot use __bio_add_page(), which doesn't do any merge. * * Meanwhile for scrub_submit_initial_read() we fully rely on the merge * to create the minimal amount of bio vectors, for fs block size < page * size cases. */ ASSERT(ret == fs_info->sectorsize); } static struct btrfs_bio *alloc_scrub_bbio(struct btrfs_fs_info *fs_info, unsigned int nr_vecs, blk_opf_t opf, u64 logical, btrfs_bio_end_io_t end_io, void *private) { struct btrfs_bio *bbio; bbio = btrfs_bio_alloc(nr_vecs, opf, BTRFS_I(fs_info->btree_inode), logical, end_io, private); bbio->is_scrub = true; bbio->bio.bi_iter.bi_sector = logical >> SECTOR_SHIFT; return bbio; } static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe, int mirror, int blocksize, bool wait) { struct btrfs_fs_info *fs_info = stripe->bg->fs_info; struct btrfs_bio *bbio = NULL; const unsigned long old_error_bitmap = scrub_bitmap_read_error(stripe); int i; ASSERT(stripe->mirror_num >= 1, "stripe->mirror_num=%d", stripe->mirror_num); ASSERT(atomic_read(&stripe->pending_io) == 0, "atomic_read(&stripe->pending_io)=%d", atomic_read(&stripe->pending_io)); for_each_set_bit(i, &old_error_bitmap, stripe->nr_sectors) { /* The current sector cannot be merged, submit the bio. */ if (bbio && ((i > 0 && !test_bit(i - 1, &old_error_bitmap)) || bbio->bio.bi_iter.bi_size >= blocksize)) { ASSERT(bbio->bio.bi_iter.bi_size); atomic_inc(&stripe->pending_io); btrfs_submit_bbio(bbio, mirror); if (wait) wait_scrub_stripe_io(stripe); bbio = NULL; } if (!bbio) bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_READ, stripe->logical + (i << fs_info->sectorsize_bits), scrub_repair_read_endio, stripe); scrub_bio_add_sector(bbio, stripe, i); } if (bbio) { ASSERT(bbio->bio.bi_iter.bi_size); atomic_inc(&stripe->pending_io); btrfs_submit_bbio(bbio, mirror); if (wait) wait_scrub_stripe_io(stripe); } } static void scrub_stripe_report_errors(struct scrub_ctx *sctx, struct scrub_stripe *stripe, const struct scrub_error_records *errors) { static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); struct btrfs_fs_info *fs_info = sctx->fs_info; struct btrfs_device *dev = NULL; const unsigned long extent_bitmap = scrub_bitmap_read_has_extent(stripe); const unsigned long error_bitmap = scrub_bitmap_read_error(stripe); u64 physical = 0; int nr_data_sectors = 0; int nr_meta_sectors = 0; int nr_nodatacsum_sectors = 0; int nr_repaired_sectors = 0; int sector_nr; if (test_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state)) return; /* * Init needed infos for error reporting. * * Although our scrub_stripe infrastructure is mostly based on btrfs_submit_bio() * thus no need for dev/physical, error reporting still needs dev and physical. */ if (!bitmap_empty(&errors->init_error_bitmap, stripe->nr_sectors)) { u64 mapped_len = fs_info->sectorsize; struct btrfs_io_context *bioc = NULL; int stripe_index = stripe->mirror_num - 1; int ret; /* For scrub, our mirror_num should always start at 1. */ ASSERT(stripe->mirror_num >= 1, "stripe->mirror_num=%d", stripe->mirror_num); ret = btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS, stripe->logical, &mapped_len, &bioc, NULL, NULL); /* * If we failed, dev will be NULL, and later detailed reports * will just be skipped. */ if (ret < 0) goto skip; physical = bioc->stripes[stripe_index].physical; dev = bioc->stripes[stripe_index].dev; btrfs_put_bioc(bioc); } skip: for_each_set_bit(sector_nr, &extent_bitmap, stripe->nr_sectors) { bool repaired = false; if (scrub_bitmap_test_bit_is_metadata(stripe, sector_nr)) { nr_meta_sectors++; } else { nr_data_sectors++; if (!stripe->sectors[sector_nr].csum) nr_nodatacsum_sectors++; } if (test_bit(sector_nr, &errors->init_error_bitmap) && !test_bit(sector_nr, &error_bitmap)) { nr_repaired_sectors++; repaired = true; } /* Good sector from the beginning, nothing need to be done. */ if (!test_bit(sector_nr, &errors->init_error_bitmap)) continue; /* * Report error for the corrupted sectors. If repaired, just * output the message of repaired message. */ if (repaired) { if (dev) { btrfs_err_rl(fs_info, "scrub: fixed up error at logical %llu on dev %s physical %llu", stripe->logical, btrfs_dev_name(dev), physical); } else { btrfs_err_rl(fs_info, "scrub: fixed up error at logical %llu on mirror %u", stripe->logical, stripe->mirror_num); } continue; } /* The remaining are all for unrepaired. */ if (dev) { btrfs_err_rl(fs_info, "scrub: unable to fixup (regular) error at logical %llu on dev %s physical %llu", stripe->logical, btrfs_dev_name(dev), physical); } else { btrfs_err_rl(fs_info, "scrub: unable to fixup (regular) error at logical %llu on mirror %u", stripe->logical, stripe->mirror_num); } if (scrub_bitmap_test_bit_io_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("i/o error", dev, false, stripe->logical, physical); if (scrub_bitmap_test_bit_csum_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("checksum error", dev, false, stripe->logical, physical); if (scrub_bitmap_test_bit_meta_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("header error", dev, false, stripe->logical, physical); if (scrub_bitmap_test_bit_meta_gen_error(stripe, sector_nr)) if (__ratelimit(&rs) && dev) scrub_print_common_warning("generation error", dev, false, stripe->logical, physical); } /* Update the device stats. */ for (int i = 0; i < errors->nr_io_errors; i++) btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_READ_ERRS); for (int i = 0; i < errors->nr_csum_errors; i++) btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_CORRUPTION_ERRS); /* Generation mismatch error is based on each metadata, not each block. */ for (int i = 0; i < errors->nr_meta_gen_errors; i += (fs_info->nodesize >> fs_info->sectorsize_bits)) btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_GENERATION_ERRS); spin_lock(&sctx->stat_lock); sctx->stat.data_extents_scrubbed += stripe->nr_data_extents; sctx->stat.tree_extents_scrubbed += stripe->nr_meta_extents; sctx->stat.data_bytes_scrubbed += nr_data_sectors << fs_info->sectorsize_bits; sctx->stat.tree_bytes_scrubbed += nr_meta_sectors << fs_info->sectorsize_bits; sctx->stat.no_csum += nr_nodatacsum_sectors; sctx->stat.read_errors += errors->nr_io_errors; sctx->stat.csum_errors += errors->nr_csum_errors; sctx->stat.verify_errors += errors->nr_meta_errors + errors->nr_meta_gen_errors; sctx->stat.uncorrectable_errors += bitmap_weight(&error_bitmap, stripe->nr_sectors); sctx->stat.corrected_errors += nr_repaired_sectors; spin_unlock(&sctx->stat_lock); } static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe, unsigned long write_bitmap, bool dev_replace); /* * The main entrance for all read related scrub work, including: * * - Wait for the initial read to finish * - Verify and locate any bad sectors * - Go through the remaining mirrors and try to read as large blocksize as * possible * - Go through all mirrors (including the failed mirror) sector-by-sector * - Submit writeback for repaired sectors * * Writeback for dev-replace does not happen here, it needs extra * synchronization for zoned devices. */ static void scrub_stripe_read_repair_worker(struct work_struct *work) { struct scrub_stripe *stripe = container_of(work, struct scrub_stripe, work); struct scrub_ctx *sctx = stripe->sctx; struct btrfs_fs_info *fs_info = sctx->fs_info; struct scrub_error_records errors = { 0 }; int num_copies = btrfs_num_copies(fs_info, stripe->bg->start, stripe->bg->length); unsigned long repaired; unsigned long error; int mirror; int i; ASSERT(stripe->mirror_num >= 1, "stripe->mirror_num=%d", stripe->mirror_num); wait_scrub_stripe_io(stripe); scrub_verify_one_stripe(stripe, scrub_bitmap_read_has_extent(stripe)); /* Save the initial failed bitmap for later repair and report usage. */ errors.init_error_bitmap = scrub_bitmap_read_error(stripe); errors.nr_io_errors = scrub_bitmap_weight_io_error(stripe); errors.nr_csum_errors = scrub_bitmap_weight_csum_error(stripe); errors.nr_meta_errors = scrub_bitmap_weight_meta_error(stripe); errors.nr_meta_gen_errors = scrub_bitmap_weight_meta_gen_error(stripe); if (bitmap_empty(&errors.init_error_bitmap, stripe->nr_sectors)) goto out; /* * Try all remaining mirrors. * * Here we still try to read as large block as possible, as this is * faster and we have extra safety nets to rely on. */ for (mirror = calc_next_mirror(stripe->mirror_num, num_copies); mirror != stripe->mirror_num; mirror = calc_next_mirror(mirror, num_copies)) { const unsigned long old_error_bitmap = scrub_bitmap_read_error(stripe); scrub_stripe_submit_repair_read(stripe, mirror, BTRFS_STRIPE_LEN, false); wait_scrub_stripe_io(stripe); scrub_verify_one_stripe(stripe, old_error_bitmap); if (scrub_bitmap_empty_error(stripe)) goto out; } /* * Last safety net, try re-checking all mirrors, including the failed * one, sector-by-sector. * * As if one sector failed the drive's internal csum, the whole read * containing the offending sector would be marked as error. * Thus here we do sector-by-sector read. * * This can be slow, thus we only try it as the last resort. */ for (i = 0, mirror = stripe->mirror_num; i < num_copies; i++, mirror = calc_next_mirror(mirror, num_copies)) { const unsigned long old_error_bitmap = scrub_bitmap_read_error(stripe); scrub_stripe_submit_repair_read(stripe, mirror, fs_info->sectorsize, true); wait_scrub_stripe_io(stripe); scrub_verify_one_stripe(stripe, old_error_bitmap); if (scrub_bitmap_empty_error(stripe)) goto out; } out: error = scrub_bitmap_read_error(stripe); /* * Submit the repaired sectors. For zoned case, we cannot do repair * in-place, but queue the bg to be relocated. */ bitmap_andnot(&repaired, &errors.init_error_bitmap, &error, stripe->nr_sectors); if (!sctx->readonly && !bitmap_empty(&repaired, stripe->nr_sectors)) { if (btrfs_is_zoned(fs_info)) { btrfs_repair_one_zone(fs_info, sctx->stripes[0].bg->start); } else { scrub_write_sectors(sctx, stripe, repaired, false); wait_scrub_stripe_io(stripe); } } scrub_stripe_report_errors(sctx, stripe, &errors); set_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state); wake_up(&stripe->repair_wait); } static void scrub_read_endio(struct btrfs_bio *bbio) { struct scrub_stripe *stripe = bbio->private; int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio)); int num_sectors; const u32 bio_size = bio_get_size(&bbio->bio); ASSERT(sector_nr < stripe->nr_sectors); num_sectors = bio_size >> stripe->bg->fs_info->sectorsize_bits; if (bbio->bio.bi_status) { scrub_bitmap_set_io_error(stripe, sector_nr, num_sectors); scrub_bitmap_set_error(stripe, sector_nr, num_sectors); } else { scrub_bitmap_clear_io_error(stripe, sector_nr, num_sectors); } bio_put(&bbio->bio); if (atomic_dec_and_test(&stripe->pending_io)) { wake_up(&stripe->io_wait); INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker); queue_work(stripe->bg->fs_info->scrub_workers, &stripe->work); } } static void scrub_write_endio(struct btrfs_bio *bbio) { struct scrub_stripe *stripe = bbio->private; struct btrfs_fs_info *fs_info = stripe->bg->fs_info; int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio)); const u32 bio_size = bio_get_size(&bbio->bio); if (bbio->bio.bi_status) { unsigned long flags; spin_lock_irqsave(&stripe->write_error_lock, flags); bitmap_set(&stripe->write_error_bitmap, sector_nr, bio_size >> fs_info->sectorsize_bits); spin_unlock_irqrestore(&stripe->write_error_lock, flags); for (int i = 0; i < (bio_size >> fs_info->sectorsize_bits); i++) btrfs_dev_stat_inc_and_print(stripe->dev, BTRFS_DEV_STAT_WRITE_ERRS); } bio_put(&bbio->bio); if (atomic_dec_and_test(&stripe->pending_io)) wake_up(&stripe->io_wait); } static void scrub_submit_write_bio(struct scrub_ctx *sctx, struct scrub_stripe *stripe, struct btrfs_bio *bbio, bool dev_replace) { struct btrfs_fs_info *fs_info = sctx->fs_info; u32 bio_len = bbio->bio.bi_iter.bi_size; u32 bio_off = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT) - stripe->logical; fill_writer_pointer_gap(sctx, stripe->physical + bio_off); atomic_inc(&stripe->pending_io); btrfs_submit_repair_write(bbio, stripe->mirror_num, dev_replace); if (!btrfs_is_zoned(fs_info)) return; /* * For zoned writeback, queue depth must be 1, thus we must wait for * the write to finish before the next write. */ wait_scrub_stripe_io(stripe); /* * And also need to update the write pointer if write finished * successfully. */ if (!test_bit(bio_off >> fs_info->sectorsize_bits, &stripe->write_error_bitmap)) sctx->write_pointer += bio_len; } /* * Submit the write bio(s) for the sectors specified by @write_bitmap. * * Here we utilize btrfs_submit_repair_write(), which has some extra benefits: * * - Only needs logical bytenr and mirror_num * Just like the scrub read path * * - Would only result in writes to the specified mirror * Unlike the regular writeback path, which would write back to all stripes * * - Handle dev-replace and read-repair writeback differently */ static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe, unsigned long write_bitmap, bool dev_replace) { struct btrfs_fs_info *fs_info = stripe->bg->fs_info; struct btrfs_bio *bbio = NULL; int sector_nr; for_each_set_bit(sector_nr, &write_bitmap, stripe->nr_sectors) { /* We should only writeback sectors covered by an extent. */ ASSERT(scrub_bitmap_test_bit_has_extent(stripe, sector_nr)); /* Cannot merge with previous sector, submit the current one. */ if (bbio && sector_nr && !test_bit(sector_nr - 1, &write_bitmap)) { scrub_submit_write_bio(sctx, stripe, bbio, dev_replace); bbio = NULL; } if (!bbio) bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_WRITE, stripe->logical + (sector_nr << fs_info->sectorsize_bits), scrub_write_endio, stripe); scrub_bio_add_sector(bbio, stripe, sector_nr); } if (bbio) scrub_submit_write_bio(sctx, stripe, bbio, dev_replace); } /* * Throttling of IO submission, bandwidth-limit based, the timeslice is 1 * second. Limit can be set via /sys/fs/UUID/devinfo/devid/scrub_speed_max. */ static void scrub_throttle_dev_io(struct scrub_ctx *sctx, struct btrfs_device *device, unsigned int bio_size) { const int time_slice = 1000; s64 delta; ktime_t now; u32 div; u64 bwlimit; bwlimit = READ_ONCE(device->scrub_speed_max); if (bwlimit == 0) return; /* * Slice is divided into intervals when the IO is submitted, adjust by * bwlimit and maximum of 64 intervals. */ div = clamp(bwlimit / (16 * 1024 * 1024), 1, 64); /* Start new epoch, set deadline */ now = ktime_get(); if (sctx->throttle_deadline == 0) { sctx->throttle_deadline = ktime_add_ms(now, time_slice / div); sctx->throttle_sent = 0; } /* Still in the time to send? */ if (ktime_before(now, sctx->throttle_deadline)) { /* If current bio is within the limit, send it */ sctx->throttle_sent += bio_size; if (sctx->throttle_sent <= div_u64(bwlimit, div)) return; /* We're over the limit, sleep until the rest of the slice */ delta = ktime_ms_delta(sctx->throttle_deadline, now); } else { /* New request after deadline, start new epoch */ delta = 0; } if (delta) { long timeout; timeout = div_u64(delta * HZ, 1000); schedule_timeout_interruptible(timeout); } /* Next call will start the deadline period */ sctx->throttle_deadline = 0; } /* * Given a physical address, this will calculate it's * logical offset. if this is a parity stripe, it will return * the most left data stripe's logical offset. * * return 0 if it is a data stripe, 1 means parity stripe. */ static int get_raid56_logic_offset(u64 physical, int num, struct btrfs_chunk_map *map, u64 *offset, u64 *stripe_start) { int i; int j = 0; u64 last_offset; const int data_stripes = nr_data_stripes(map); last_offset = (physical - map->stripes[num].physical) * data_stripes; if (stripe_start) *stripe_start = last_offset; *offset = last_offset; for (i = 0; i < data_stripes; i++) { u32 stripe_nr; u32 stripe_index; u32 rot; *offset = last_offset + btrfs_stripe_nr_to_offset(i); stripe_nr = (u32)(*offset >> BTRFS_STRIPE_LEN_SHIFT) / data_stripes; /* Work out the disk rotation on this stripe-set */ rot = stripe_nr % map->num_stripes; /* calculate which stripe this data locates */ rot += i; stripe_index = rot % map->num_stripes; if (stripe_index == num) return 0; if (stripe_index < num) j++; } *offset = last_offset + btrfs_stripe_nr_to_offset(j); return 1; } /* * Return 0 if the extent item range covers any byte of the range. * Return <0 if the extent item is before @search_start. * Return >0 if the extent item is after @start_start + @search_len. */ static int compare_extent_item_range(struct btrfs_path *path, u64 search_start, u64 search_len) { struct btrfs_fs_info *fs_info = path->nodes[0]->fs_info; u64 len; struct btrfs_key key; btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); ASSERT(key.type == BTRFS_EXTENT_ITEM_KEY || key.type == BTRFS_METADATA_ITEM_KEY, "key.type=%u", key.type); if (key.type == BTRFS_METADATA_ITEM_KEY) len = fs_info->nodesize; else len = key.offset; if (key.objectid + len <= search_start) return -1; if (key.objectid >= search_start + search_len) return 1; return 0; } /* * Locate one extent item which covers any byte in range * [@search_start, @search_start + @search_length) * * If the path is not initialized, we will initialize the search by doing * a btrfs_search_slot(). * If the path is already initialized, we will use the path as the initial * slot, to avoid duplicated btrfs_search_slot() calls. * * NOTE: If an extent item starts before @search_start, we will still * return the extent item. This is for data extent crossing stripe boundary. * * Return 0 if we found such extent item, and @path will point to the extent item. * Return >0 if no such extent item can be found, and @path will be released. * Return <0 if hit fatal error, and @path will be released. */ static int find_first_extent_item(struct btrfs_root *extent_root, struct btrfs_path *path, u64 search_start, u64 search_len) { struct btrfs_fs_info *fs_info = extent_root->fs_info; struct btrfs_key key; int ret; /* Continue using the existing path */ if (path->nodes[0]) goto search_forward; key.objectid = search_start; if (btrfs_fs_incompat(fs_info, SKINNY_METADATA)) key.type = BTRFS_METADATA_ITEM_KEY; else key.type = BTRFS_EXTENT_ITEM_KEY; key.offset = (u64)-1; ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); if (ret < 0) return ret; if (unlikely(ret == 0)) { /* * Key with offset -1 found, there would have to exist an extent * item with such offset, but this is out of the valid range. */ btrfs_release_path(path); return -EUCLEAN; } /* * Here we intentionally pass 0 as @min_objectid, as there could be * an extent item starting before @search_start. */ ret = btrfs_previous_extent_item(extent_root, path, 0); if (ret < 0) return ret; /* * No matter whether we have found an extent item, the next loop will * properly do every check on the key. */ search_forward: while (true) { btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); if (key.objectid >= search_start + search_len) break; if (key.type != BTRFS_METADATA_ITEM_KEY && key.type != BTRFS_EXTENT_ITEM_KEY) goto next; ret = compare_extent_item_range(path, search_start, search_len); if (ret == 0) return ret; if (ret > 0) break; next: ret = btrfs_next_item(extent_root, path); if (ret) { /* Either no more items or a fatal error. */ btrfs_release_path(path); return ret; } } btrfs_release_path(path); return 1; } static void get_extent_info(struct btrfs_path *path, u64 *extent_start_ret, u64 *size_ret, u64 *flags_ret, u64 *generation_ret) { struct btrfs_key key; struct btrfs_extent_item *ei; btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); ASSERT(key.type == BTRFS_METADATA_ITEM_KEY || key.type == BTRFS_EXTENT_ITEM_KEY, "key.type=%u", key.type); *extent_start_ret = key.objectid; if (key.type == BTRFS_METADATA_ITEM_KEY) *size_ret = path->nodes[0]->fs_info->nodesize; else *size_ret = key.offset; ei = btrfs_item_ptr(path->nodes[0], path->slots[0], struct btrfs_extent_item); *flags_ret = btrfs_extent_flags(path->nodes[0], ei); *generation_ret = btrfs_extent_generation(path->nodes[0], ei); } static int sync_write_pointer_for_zoned(struct scrub_ctx *sctx, u64 logical, u64 physical, u64 physical_end) { struct btrfs_fs_info *fs_info = sctx->fs_info; int ret = 0; if (!btrfs_is_zoned(fs_info)) return 0; mutex_lock(&sctx->wr_lock); if (sctx->write_pointer < physical_end) { ret = btrfs_sync_zone_write_pointer(sctx->wr_tgtdev, logical, physical, sctx->write_pointer); if (ret) btrfs_err(fs_info, "scrub: zoned: failed to recover write pointer"); } mutex_unlock(&sctx->wr_lock); btrfs_dev_clear_zone_empty(sctx->wr_tgtdev, physical); return ret; } static void fill_one_extent_info(struct btrfs_fs_info *fs_info, struct scrub_stripe *stripe, u64 extent_start, u64 extent_len, u64 extent_flags, u64 extent_gen) { for (u64 cur_logical = max(stripe->logical, extent_start); cur_logical < min(stripe->logical + BTRFS_STRIPE_LEN, extent_start + extent_len); cur_logical += fs_info->sectorsize) { const int nr_sector = (cur_logical - stripe->logical) >> fs_info->sectorsize_bits; struct scrub_sector_verification *sector = &stripe->sectors[nr_sector]; scrub_bitmap_set_bit_has_extent(stripe, nr_sector); if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) { scrub_bitmap_set_bit_is_metadata(stripe, nr_sector); sector->generation = extent_gen; } } } static void scrub_stripe_reset_bitmaps(struct scrub_stripe *stripe) { ASSERT(stripe->nr_sectors); bitmap_zero(stripe->bitmaps, scrub_bitmap_nr_last * stripe->nr_sectors); } /* * Locate one stripe which has at least one extent in its range. * * Return 0 if found such stripe, and store its info into @stripe. * Return >0 if there is no such stripe in the specified range. * Return <0 for error. */ static int scrub_find_fill_first_stripe(struct btrfs_block_group *bg, struct btrfs_path *extent_path, struct btrfs_path *csum_path, struct btrfs_device *dev, u64 physical, int mirror_num, u64 logical_start, u32 logical_len, struct scrub_stripe *stripe) { struct btrfs_fs_info *fs_info = bg->fs_info; struct btrfs_root *extent_root = btrfs_extent_root(fs_info, bg->start); struct btrfs_root *csum_root = btrfs_csum_root(fs_info, bg->start); const u64 logical_end = logical_start + logical_len; u64 cur_logical = logical_start; u64 stripe_end; u64 extent_start; u64 extent_len; u64 extent_flags; u64 extent_gen; int ret; if (unlikely(!extent_root || !csum_root)) { btrfs_err(fs_info, "scrub: no valid extent or csum root found"); return -EUCLEAN; } memset(stripe->sectors, 0, sizeof(struct scrub_sector_verification) * stripe->nr_sectors); scrub_stripe_reset_bitmaps(stripe); /* The range must be inside the bg. */ ASSERT(logical_start >= bg->start && logical_end <= btrfs_block_group_end(bg), "bg->start=%llu logical_start=%llu logical_end=%llu end=%llu", bg->start, logical_start, logical_end, btrfs_block_group_end(bg)); ret = find_first_extent_item(extent_root, extent_path, logical_start, logical_len); /* Either error or not found. */ if (ret) return ret; get_extent_info(extent_path, &extent_start, &extent_len, &extent_flags, &extent_gen); if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) stripe->nr_meta_extents++; if (extent_flags & BTRFS_EXTENT_FLAG_DATA) stripe->nr_data_extents++; cur_logical = max(extent_start, cur_logical); /* * Round down to stripe boundary. * * The extra calculation against bg->start is to handle block groups * whose logical bytenr is not BTRFS_STRIPE_LEN aligned. */ stripe->logical = round_down(cur_logical - bg->start, BTRFS_STRIPE_LEN) + bg->start; stripe->physical = physical + stripe->logical - logical_start; stripe->dev = dev; stripe->bg = bg; stripe->mirror_num = mirror_num; stripe_end = stripe->logical + BTRFS_STRIPE_LEN - 1; /* Fill the first extent info into stripe->sectors[] array. */ fill_one_extent_info(fs_info, stripe, extent_start, extent_len, extent_flags, extent_gen); cur_logical = extent_start + extent_len; /* Fill the extent info for the remaining sectors. */ while (cur_logical <= stripe_end) { ret = find_first_extent_item(extent_root, extent_path, cur_logical, stripe_end - cur_logical + 1); if (ret < 0) return ret; if (ret > 0) { ret = 0; break; } get_extent_info(extent_path, &extent_start, &extent_len, &extent_flags, &extent_gen); if (extent_flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) stripe->nr_meta_extents++; if (extent_flags & BTRFS_EXTENT_FLAG_DATA) stripe->nr_data_extents++; fill_one_extent_info(fs_info, stripe, extent_start, extent_len, extent_flags, extent_gen); cur_logical = extent_start + extent_len; } /* Now fill the data csum. */ if (bg->flags & BTRFS_BLOCK_GROUP_DATA) { int sector_nr; unsigned long csum_bitmap = 0; /* Csum space should have already been allocated. */ ASSERT(stripe->csums); /* * Our csum bitmap should be large enough, as BTRFS_STRIPE_LEN * should contain at most 16 sectors. */ ASSERT(BITS_PER_LONG >= BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits); ret = btrfs_lookup_csums_bitmap(csum_root, csum_path, stripe->logical, stripe_end, stripe->csums, &csum_bitmap); if (ret < 0) return ret; if (ret > 0) ret = 0; for_each_set_bit(sector_nr, &csum_bitmap, stripe->nr_sectors) { stripe->sectors[sector_nr].csum = stripe->csums + sector_nr * fs_info->csum_size; } } set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state); return ret; } static void scrub_reset_stripe(struct scrub_stripe *stripe) { scrub_stripe_reset_bitmaps(stripe); stripe->nr_meta_extents = 0; stripe->nr_data_extents = 0; stripe->state = 0; for (int i = 0; i < stripe->nr_sectors; i++) { stripe->sectors[i].csum = NULL; stripe->sectors[i].generation = 0; } } static u32 stripe_length(const struct scrub_stripe *stripe) { ASSERT(stripe->bg); return min(BTRFS_STRIPE_LEN, stripe->bg->start + stripe->bg->length - stripe->logical); } static void scrub_submit_extent_sector_read(struct scrub_stripe *stripe) { struct btrfs_fs_info *fs_info = stripe->bg->fs_info; struct btrfs_bio *bbio = NULL; unsigned int nr_sectors = stripe_length(stripe) >> fs_info->sectorsize_bits; const unsigned long has_extent = scrub_bitmap_read_has_extent(stripe); u64 stripe_len = BTRFS_STRIPE_LEN; int mirror = stripe->mirror_num; int i; atomic_inc(&stripe->pending_io); for_each_set_bit(i, &has_extent, stripe->nr_sectors) { /* We're beyond the chunk boundary, no need to read anymore. */ if (i >= nr_sectors) break; /* The current sector cannot be merged, submit the bio. */ if (bbio && ((i > 0 && !test_bit(i - 1, &has_extent)) || bbio->bio.bi_iter.bi_size >= stripe_len)) { ASSERT(bbio->bio.bi_iter.bi_size); atomic_inc(&stripe->pending_io); btrfs_submit_bbio(bbio, mirror); bbio = NULL; } if (!bbio) { struct btrfs_io_stripe io_stripe = {}; struct btrfs_io_context *bioc = NULL; const u64 logical = stripe->logical + (i << fs_info->sectorsize_bits); int ret; io_stripe.rst_search_commit_root = true; stripe_len = (nr_sectors - i) << fs_info->sectorsize_bits; /* * For RST cases, we need to manually split the bbio to * follow the RST boundary. */ ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, logical, &stripe_len, &bioc, &io_stripe, &mirror); btrfs_put_bioc(bioc); if (ret < 0) { if (ret != -ENODATA) { /* * Earlier btrfs_get_raid_extent_offset() * returned -ENODATA, which means there's * no entry for the corresponding range * in the stripe tree. But if it's in * the extent tree, then it's a preallocated * extent and not an error. */ scrub_bitmap_set_bit_io_error(stripe, i); scrub_bitmap_set_bit_error(stripe, i); } continue; } bbio = alloc_scrub_bbio(fs_info, stripe->nr_sectors, REQ_OP_READ, logical, scrub_read_endio, stripe); } scrub_bio_add_sector(bbio, stripe, i); } if (bbio) { ASSERT(bbio->bio.bi_iter.bi_size); atomic_inc(&stripe->pending_io); btrfs_submit_bbio(bbio, mirror); } if (atomic_dec_and_test(&stripe->pending_io)) { wake_up(&stripe->io_wait); INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker); queue_work(stripe->bg->fs_info->scrub_workers, &stripe->work); } } static void scrub_submit_initial_read(struct scrub_ctx *sctx, struct scrub_stripe *stripe) { struct btrfs_fs_info *fs_info = sctx->fs_info; struct btrfs_bio *bbio; const u32 min_folio_shift = PAGE_SHIFT + fs_info->block_min_order; unsigned int nr_sectors = stripe_length(stripe) >> fs_info->sectorsize_bits; int mirror = stripe->mirror_num; ASSERT(stripe->bg); ASSERT(stripe->mirror_num > 0); ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state)); if (btrfs_need_stripe_tree_update(fs_info, stripe->bg->flags)) { scrub_submit_extent_sector_read(stripe); return; } bbio = alloc_scrub_bbio(fs_info, BTRFS_STRIPE_LEN >> min_folio_shift, REQ_OP_READ, stripe->logical, scrub_read_endio, stripe); /* Read the whole range inside the chunk boundary. */ for (unsigned int cur = 0; cur < nr_sectors; cur++) scrub_bio_add_sector(bbio, stripe, cur); atomic_inc(&stripe->pending_io); /* * For dev-replace, either user asks to avoid the source dev, or * the device is missing, we try the next mirror instead. */ if (sctx->is_dev_replace && (fs_info->dev_replace.cont_reading_from_srcdev_mode == BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID || !stripe->dev->bdev)) { int num_copies = btrfs_num_copies(fs_info, stripe->bg->start, stripe->bg->length); mirror = calc_next_mirror(mirror, num_copies); } btrfs_submit_bbio(bbio, mirror); } static bool stripe_has_metadata_error(struct scrub_stripe *stripe) { const unsigned long error = scrub_bitmap_read_error(stripe); int i; for_each_set_bit(i, &error, stripe->nr_sectors) { if (scrub_bitmap_test_bit_is_metadata(stripe, i)) { struct btrfs_fs_info *fs_info = stripe->bg->fs_info; btrfs_err(fs_info, "scrub: stripe %llu has unrepaired metadata sector at logical %llu", stripe->logical, stripe->logical + (i << fs_info->sectorsize_bits)); return true; } } return false; } static void submit_initial_group_read(struct scrub_ctx *sctx, unsigned int first_slot, unsigned int nr_stripes) { struct blk_plug plug; ASSERT(first_slot < SCRUB_TOTAL_STRIPES); ASSERT(first_slot + nr_stripes <= SCRUB_TOTAL_STRIPES); scrub_throttle_dev_io(sctx, sctx->stripes[0].dev, btrfs_stripe_nr_to_offset(nr_stripes)); blk_start_plug(&plug); for (int i = 0; i < nr_stripes; i++) { struct scrub_stripe *stripe = &sctx->stripes[first_slot + i]; /* Those stripes should be initialized. */ ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state)); scrub_submit_initial_read(sctx, stripe); } blk_finish_plug(&plug); } static int flush_scrub_stripes(struct scrub_ctx *sctx) { struct btrfs_fs_info *fs_info = sctx->fs_info; struct scrub_stripe *stripe; const int nr_stripes = sctx->cur_stripe; int ret = 0; if (!nr_stripes) return 0; ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &sctx->stripes[0].state)); /* Submit the stripes which are populated but not submitted. */ if (nr_stripes % SCRUB_STRIPES_PER_GROUP) { const int first_slot = round_down(nr_stripes, SCRUB_STRIPES_PER_GROUP); submit_initial_group_read(sctx, first_slot, nr_stripes - first_slot); } for (int i = 0; i < nr_stripes; i++) { stripe = &sctx->stripes[i]; wait_event(stripe->repair_wait, test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state)); } /* Submit for dev-replace. */ if (sctx->is_dev_replace) { /* * For dev-replace, if we know there is something wrong with * metadata, we should immediately abort. */ for (int i = 0; i < nr_stripes; i++) { if (unlikely(stripe_has_metadata_error(&sctx->stripes[i]))) { ret = -EIO; goto out; } } for (int i = 0; i < nr_stripes; i++) { unsigned long good; unsigned long has_extent; unsigned long error; stripe = &sctx->stripes[i]; ASSERT(stripe->dev == fs_info->dev_replace.srcdev); has_extent = scrub_bitmap_read_has_extent(stripe); error = scrub_bitmap_read_error(stripe); bitmap_andnot(&good, &has_extent, &error, stripe->nr_sectors); scrub_write_sectors(sctx, stripe, good, true); } } /* Wait for the above writebacks to finish. */ for (int i = 0; i < nr_stripes; i++) { stripe = &sctx->stripes[i]; wait_scrub_stripe_io(stripe); spin_lock(&sctx->stat_lock); sctx->stat.last_physical = stripe->physical + stripe_length(stripe); spin_unlock(&sctx->stat_lock); scrub_reset_stripe(stripe); } out: sctx->cur_stripe = 0; return ret; } static void raid56_scrub_wait_endio(struct bio *bio) { complete(bio->bi_private); } static int queue_scrub_stripe(struct scrub_ctx *sctx, struct btrfs_block_group *bg, struct btrfs_device *dev, int mirror_num, u64 logical, u32 length, u64 physical, u64 *found_logical_ret) { struct scrub_stripe *stripe; int ret; /* * There should always be one slot left, as caller filling the last * slot should flush them all. */ ASSERT(sctx->cur_stripe < SCRUB_TOTAL_STRIPES); /* @found_logical_ret must be specified. */ ASSERT(found_logical_ret); stripe = &sctx->stripes[sctx->cur_stripe]; scrub_reset_stripe(stripe); ret = scrub_find_fill_first_stripe(bg, &sctx->extent_path, &sctx->csum_path, dev, physical, mirror_num, logical, length, stripe); /* Either >0 as no more extents or <0 for error. */ if (ret) return ret; *found_logical_ret = stripe->logical; sctx->cur_stripe++; /* We filled one group, submit it. */ if (sctx->cur_stripe % SCRUB_STRIPES_PER_GROUP == 0) { const int first_slot = sctx->cur_stripe - SCRUB_STRIPES_PER_GROUP; submit_initial_group_read(sctx, first_slot, SCRUB_STRIPES_PER_GROUP); } /* Last slot used, flush them all. */ if (sctx->cur_stripe == SCRUB_TOTAL_STRIPES) return flush_scrub_stripes(sctx); return 0; } /* * Return 0 if we should not cancel the scrub. * Return <0 if we need to cancel the scrub, returned value will * indicate the reason: * - -ECANCELED - Being explicitly canceled through ioctl. * - -EINTR - Being interrupted by signal or fs/process freezing. */ static int should_cancel_scrub(const struct scrub_ctx *sctx) { struct btrfs_fs_info *fs_info = sctx->fs_info; if (atomic_read(&fs_info->scrub_cancel_req) || atomic_read(&sctx->cancel_req)) return -ECANCELED; /* * The user (e.g. fsfreeze command) or power management (PM) * suspend/hibernate can freeze the fs. And PM suspend/hibernate will * also freeze all user processes. * * A user process can only be frozen when it is in user space, thus we * have to cancel the run so that the process can return to the user * space. * * Furthermore we have to check both filesystem and process freezing, * as PM can be configured to freeze the filesystems before processes. * * If we only check fs freezing, then suspend without fs freezing * will timeout, as the process is still in kernel space. * * If we only check process freezing, then suspend with fs freezing * will timeout, as the running scrub will prevent the fs from being frozen. */ if (fs_info->sb->s_writers.frozen > SB_UNFROZEN || freezing(current) || signal_pending(current)) return -EINTR; return 0; } static int scrub_raid56_cached_parity(struct scrub_ctx *sctx, struct btrfs_device *scrub_dev, struct btrfs_chunk_map *map, u64 full_stripe_start, unsigned long *extent_bitmap) { DECLARE_COMPLETION_ONSTACK(io_done); struct btrfs_fs_info *fs_info = sctx->fs_info; struct btrfs_io_context *bioc = NULL; struct btrfs_raid_bio *rbio; struct bio bio; const int data_stripes = nr_data_stripes(map); u64 length = btrfs_stripe_nr_to_offset(data_stripes); int ret; bio_init(&bio, NULL, NULL, 0, REQ_OP_READ); bio.bi_iter.bi_sector = full_stripe_start >> SECTOR_SHIFT; bio.bi_private = &io_done; bio.bi_end_io = raid56_scrub_wait_endio; btrfs_bio_counter_inc_blocked(fs_info); ret = btrfs_map_block(fs_info, BTRFS_MAP_WRITE, full_stripe_start, &length, &bioc, NULL, NULL); if (ret < 0) goto out; /* For RAID56 write there must be an @bioc allocated. */ ASSERT(bioc); rbio = raid56_parity_alloc_scrub_rbio(&bio, bioc, scrub_dev, extent_bitmap, BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits); btrfs_put_bioc(bioc); if (!rbio) { ret = -ENOMEM; goto out; } /* Use the recovered stripes as cache to avoid read them from disk again. */ for (int i = 0; i < data_stripes; i++) { struct scrub_stripe *stripe = &sctx->raid56_data_stripes[i]; raid56_parity_cache_data_folios(rbio, stripe->folios, full_stripe_start + (i << BTRFS_STRIPE_LEN_SHIFT)); } raid56_parity_submit_scrub_rbio(rbio); wait_for_completion_io(&io_done); ret = blk_status_to_errno(bio.bi_status); out: btrfs_bio_counter_dec(fs_info); bio_uninit(&bio); return ret; } static int scrub_raid56_parity_stripe(struct scrub_ctx *sctx, struct btrfs_device *scrub_dev, struct btrfs_block_group *bg, struct btrfs_chunk_map *map, u64 full_stripe_start) { struct btrfs_fs_info *fs_info = sctx->fs_info; BTRFS_PATH_AUTO_RELEASE(extent_path); BTRFS_PATH_AUTO_RELEASE(csum_path); struct scrub_stripe *stripe; bool all_empty = true; const int data_stripes = nr_data_stripes(map); unsigned long extent_bitmap = 0; int ret; ASSERT(sctx->raid56_data_stripes); ret = should_cancel_scrub(sctx); if (ret < 0) return ret; if (atomic_read(&fs_info->scrub_pause_req)) scrub_blocked_if_needed(fs_info); spin_lock(&bg->lock); if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags)) { spin_unlock(&bg->lock); return 0; } spin_unlock(&bg->lock); /* * For data stripe search, we cannot reuse the same extent/csum paths, * as the data stripe bytenr may be smaller than previous extent. Thus * we have to use our own extent/csum paths. */ extent_path.search_commit_root = true; extent_path.skip_locking = true; csum_path.search_commit_root = true; csum_path.skip_locking = true; for (int i = 0; i < data_stripes; i++) { int stripe_index; int rot; u64 physical; stripe = &sctx->raid56_data_stripes[i]; rot = div_u64(full_stripe_start - bg->start, data_stripes) >> BTRFS_STRIPE_LEN_SHIFT; stripe_index = (i + rot) % map->num_stripes; physical = map->stripes[stripe_index].physical + btrfs_stripe_nr_to_offset(rot); scrub_reset_stripe(stripe); set_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state); ret = scrub_find_fill_first_stripe(bg, &extent_path, &csum_path, map->stripes[stripe_index].dev, physical, 1, full_stripe_start + btrfs_stripe_nr_to_offset(i), BTRFS_STRIPE_LEN, stripe); if (ret < 0) return ret; /* * No extent in this data stripe, need to manually mark them * initialized to make later read submission happy. */ if (ret > 0) { stripe->logical = full_stripe_start + btrfs_stripe_nr_to_offset(i); stripe->dev = map->stripes[stripe_index].dev; stripe->mirror_num = 1; set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state); } } /* Check if all data stripes are empty. */ for (int i = 0; i < data_stripes; i++) { stripe = &sctx->raid56_data_stripes[i]; if (!scrub_bitmap_empty_has_extent(stripe)) { all_empty = false; break; } } if (all_empty) return 0; for (int i = 0; i < data_stripes; i++) { stripe = &sctx->raid56_data_stripes[i]; scrub_submit_initial_read(sctx, stripe); } for (int i = 0; i < data_stripes; i++) { stripe = &sctx->raid56_data_stripes[i]; wait_event(stripe->repair_wait, test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state)); } /* For now, no zoned support for RAID56. */ ASSERT(!btrfs_is_zoned(sctx->fs_info)); /* * Now all data stripes are properly verified. Check if we have any * unrepaired, if so abort immediately or we could further corrupt the * P/Q stripes. * * During the loop, also populate extent_bitmap. */ for (int i = 0; i < data_stripes; i++) { unsigned long error; unsigned long has_extent; stripe = &sctx->raid56_data_stripes[i]; error = scrub_bitmap_read_error(stripe); has_extent = scrub_bitmap_read_has_extent(stripe); /* * We should only check the errors where there is an extent. * As we may hit an empty data stripe while it's missing. */ bitmap_and(&error, &error, &has_extent, stripe->nr_sectors); if (unlikely(!bitmap_empty(&error, stripe->nr_sectors))) { btrfs_err(fs_info, "scrub: unrepaired sectors detected, full stripe %llu data stripe %u errors %*pbl", full_stripe_start, i, stripe->nr_sectors, &error); return ret; } bitmap_or(&extent_bitmap, &extent_bitmap, &has_extent, stripe->nr_sectors); } /* Now we can check and regenerate the P/Q stripe. */ return scrub_raid56_cached_parity(sctx, scrub_dev, map, full_stripe_start, &extent_bitmap); } /* * Scrub one range which can only has simple mirror based profile. * (Including all range in SINGLE/DUP/RAID1/RAID1C*, and each stripe in * RAID0/RAID10). * * Since we may need to handle a subset of block group, we need @logical_start * and @logical_length parameter. */ static int scrub_simple_mirror(struct scrub_ctx *sctx, struct btrfs_block_group *bg, u64 logical_start, u64 logical_length, struct btrfs_device *device, u64 physical, int mirror_num) { struct btrfs_fs_info *fs_info = sctx->fs_info; const u64 logical_end = logical_start + logical_length; u64 cur_logical = logical_start; int ret = 0; /* The range must be inside the bg */ ASSERT(logical_start >= bg->start && logical_end <= btrfs_block_group_end(bg)); /* Go through each extent items inside the logical range */ while (cur_logical < logical_end) { u64 found_logical = U64_MAX; u64 cur_physical = physical + cur_logical - logical_start; ret = should_cancel_scrub(sctx); if (ret < 0) break; if (atomic_read(&fs_info->scrub_pause_req)) scrub_blocked_if_needed(fs_info); spin_lock(&bg->lock); if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags)) { spin_unlock(&bg->lock); ret = 0; break; } spin_unlock(&bg->lock); ret = queue_scrub_stripe(sctx, bg, device, mirror_num, cur_logical, logical_end - cur_logical, cur_physical, &found_logical); if (ret > 0) { /* No more extent, just update the accounting */ spin_lock(&sctx->stat_lock); sctx->stat.last_physical = physical + logical_length; spin_unlock(&sctx->stat_lock); ret = 0; break; } if (ret < 0) break; /* queue_scrub_stripe() returned 0, @found_logical must be updated. */ ASSERT(found_logical != U64_MAX); cur_logical = found_logical + BTRFS_STRIPE_LEN; /* Don't hold CPU for too long time */ cond_resched(); } return ret; } /* Calculate the full stripe length for simple stripe based profiles */ static u64 simple_stripe_full_stripe_len(const struct btrfs_chunk_map *map) { ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10)); return btrfs_stripe_nr_to_offset(map->num_stripes / map->sub_stripes); } /* Get the logical bytenr for the stripe */ static u64 simple_stripe_get_logical(struct btrfs_chunk_map *map, struct btrfs_block_group *bg, int stripe_index) { ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10)); ASSERT(stripe_index < map->num_stripes); /* * (stripe_index / sub_stripes) gives how many data stripes we need to * skip. */ return btrfs_stripe_nr_to_offset(stripe_index / map->sub_stripes) + bg->start; } /* Get the mirror number for the stripe */ static int simple_stripe_mirror_num(struct btrfs_chunk_map *map, int stripe_index) { ASSERT(map->type & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10)); ASSERT(stripe_index < map->num_stripes); /* For RAID0, it's fixed to 1, for RAID10 it's 0,1,0,1... */ return stripe_index % map->sub_stripes + 1; } static int scrub_simple_stripe(struct scrub_ctx *sctx, struct btrfs_block_group *bg, struct btrfs_chunk_map *map, struct btrfs_device *device, int stripe_index) { const u64 logical_increment = simple_stripe_full_stripe_len(map); const u64 orig_logical = simple_stripe_get_logical(map, bg, stripe_index); const u64 orig_physical = map->stripes[stripe_index].physical; const u64 end = btrfs_block_group_end(bg); const int mirror_num = simple_stripe_mirror_num(map, stripe_index); u64 cur_logical = orig_logical; u64 cur_physical = orig_physical; int ret = 0; while (cur_logical < end) { /* * Inside each stripe, RAID0 is just SINGLE, and RAID10 is * just RAID1, so we can reuse scrub_simple_mirror() to scrub * this stripe. */ ret = scrub_simple_mirror(sctx, bg, cur_logical, BTRFS_STRIPE_LEN, device, cur_physical, mirror_num); if (ret) return ret; /* Skip to next stripe which belongs to the target device */ cur_logical += logical_increment; /* For physical offset, we just go to next stripe */ cur_physical += BTRFS_STRIPE_LEN; } return ret; } static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, struct btrfs_block_group *bg, struct btrfs_chunk_map *map, struct btrfs_device *scrub_dev, int stripe_index) { struct btrfs_fs_info *fs_info = sctx->fs_info; const u64 profile = map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK; const u64 chunk_logical = bg->start; int ret; int ret2; u64 physical = map->stripes[stripe_index].physical; const u64 dev_stripe_len = btrfs_calc_stripe_length(map); const u64 physical_end = physical + dev_stripe_len; u64 logical; u64 logic_end; /* The logical increment after finishing one stripe */ u64 increment; /* Offset inside the chunk */ u64 offset; u64 stripe_logical; /* Extent_path should be released by now. */ ASSERT(sctx->extent_path.nodes[0] == NULL); scrub_blocked_if_needed(fs_info); if (sctx->is_dev_replace && btrfs_dev_is_sequential(sctx->wr_tgtdev, physical)) { mutex_lock(&sctx->wr_lock); sctx->write_pointer = physical; mutex_unlock(&sctx->wr_lock); } /* Prepare the extra data stripes used by RAID56. */ if (profile & BTRFS_BLOCK_GROUP_RAID56_MASK) { ASSERT(sctx->raid56_data_stripes == NULL); sctx->raid56_data_stripes = kzalloc_objs(struct scrub_stripe, nr_data_stripes(map)); if (!sctx->raid56_data_stripes) { ret = -ENOMEM; goto out; } for (int i = 0; i < nr_data_stripes(map); i++) { ret = init_scrub_stripe(fs_info, &sctx->raid56_data_stripes[i]); if (ret < 0) goto out; sctx->raid56_data_stripes[i].bg = bg; sctx->raid56_data_stripes[i].sctx = sctx; } } /* * There used to be a big double loop to handle all profiles using the * same routine, which grows larger and more gross over time. * * So here we handle each profile differently, so simpler profiles * have simpler scrubbing function. */ if (!(profile & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10 | BTRFS_BLOCK_GROUP_RAID56_MASK))) { /* * Above check rules out all complex profile, the remaining * profiles are SINGLE|DUP|RAID1|RAID1C*, which is simple * mirrored duplication without stripe. * * Only @physical and @mirror_num needs to calculated using * @stripe_index. */ ret = scrub_simple_mirror(sctx, bg, bg->start, bg->length, scrub_dev, map->stripes[stripe_index].physical, stripe_index + 1); offset = 0; goto out; } if (profile & (BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID10)) { ret = scrub_simple_stripe(sctx, bg, map, scrub_dev, stripe_index); offset = btrfs_stripe_nr_to_offset(stripe_index / map->sub_stripes); goto out; } /* Only RAID56 goes through the old code */ ASSERT(map->type & BTRFS_BLOCK_GROUP_RAID56_MASK); ret = 0; /* Calculate the logical end of the stripe */ get_raid56_logic_offset(physical_end, stripe_index, map, &logic_end, NULL); logic_end += chunk_logical; /* Initialize @offset in case we need to go to out: label */ get_raid56_logic_offset(physical, stripe_index, map, &offset, NULL); increment = btrfs_stripe_nr_to_offset(nr_data_stripes(map)); /* * Due to the rotation, for RAID56 it's better to iterate each stripe * using their physical offset. */ while (physical < physical_end) { ret = get_raid56_logic_offset(physical, stripe_index, map, &logical, &stripe_logical); logical += chunk_logical; if (ret) { /* it is parity strip */ stripe_logical += chunk_logical; ret = scrub_raid56_parity_stripe(sctx, scrub_dev, bg, map, stripe_logical); spin_lock(&sctx->stat_lock); sctx->stat.last_physical = min(physical + BTRFS_STRIPE_LEN, physical_end); spin_unlock(&sctx->stat_lock); if (ret) goto out; goto next; } /* * Now we're at a data stripe, scrub each extents in the range. * * At this stage, if we ignore the repair part, inside each data * stripe it is no different than SINGLE profile. * We can reuse scrub_simple_mirror() here, as the repair part * is still based on @mirror_num. */ ret = scrub_simple_mirror(sctx, bg, logical, BTRFS_STRIPE_LEN, scrub_dev, physical, 1); if (ret < 0) goto out; next: logical += increment; physical += BTRFS_STRIPE_LEN; spin_lock(&sctx->stat_lock); sctx->stat.last_physical = physical; spin_unlock(&sctx->stat_lock); } out: ret2 = flush_scrub_stripes(sctx); if (!ret) ret = ret2; btrfs_release_path(&sctx->extent_path); btrfs_release_path(&sctx->csum_path); if (sctx->raid56_data_stripes) { for (int i = 0; i < nr_data_stripes(map); i++) release_scrub_stripe(&sctx->raid56_data_stripes[i]); kfree(sctx->raid56_data_stripes); sctx->raid56_data_stripes = NULL; } if (sctx->is_dev_replace && ret >= 0) { ret2 = sync_write_pointer_for_zoned(sctx, chunk_logical + offset, map->stripes[stripe_index].physical, physical_end); if (ret2) ret = ret2; } return ret < 0 ? ret : 0; } static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx, struct btrfs_block_group *bg, struct btrfs_device *scrub_dev, u64 dev_offset, u64 dev_extent_len) { struct btrfs_fs_info *fs_info = sctx->fs_info; struct btrfs_chunk_map *map; int i; int ret = 0; map = btrfs_find_chunk_map(fs_info, bg->start, bg->length); if (!map) { /* * Might have been an unused block group deleted by the cleaner * kthread or relocation. */ spin_lock(&bg->lock); if (!test_bit(BLOCK_GROUP_FLAG_REMOVED, &bg->runtime_flags)) ret = -EINVAL; spin_unlock(&bg->lock); return ret; } if (map->start != bg->start) goto out; if (map->chunk_len < dev_extent_len) goto out; for (i = 0; i < map->num_stripes; ++i) { if (map->stripes[i].dev->bdev == scrub_dev->bdev && map->stripes[i].physical == dev_offset) { ret = scrub_stripe(sctx, bg, map, scrub_dev, i); if (ret) goto out; } } out: btrfs_free_chunk_map(map); return ret; } static int finish_extent_writes_for_zoned(struct btrfs_root *root, struct btrfs_block_group *cache) { struct btrfs_fs_info *fs_info = cache->fs_info; if (!btrfs_is_zoned(fs_info)) return 0; btrfs_wait_block_group_reservations(cache); btrfs_wait_nocow_writers(cache); btrfs_wait_ordered_roots(fs_info, U64_MAX, cache); return btrfs_commit_current_transaction(root); } static noinline_for_stack int scrub_enumerate_chunks(struct scrub_ctx *sctx, struct btrfs_device *scrub_dev, u64 start, u64 end) { struct btrfs_dev_extent *dev_extent = NULL; BTRFS_PATH_AUTO_FREE(path); struct btrfs_fs_info *fs_info = sctx->fs_info; struct btrfs_root *root = fs_info->dev_root; u64 chunk_offset; int ret = 0; int ro_set; int slot; struct extent_buffer *l; struct btrfs_key key; struct btrfs_key found_key; struct btrfs_block_group *cache; struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; path = btrfs_alloc_path(); if (!path) return -ENOMEM; path->reada = READA_FORWARD; path->search_commit_root = true; path->skip_locking = true; key.objectid = scrub_dev->devid; key.type = BTRFS_DEV_EXTENT_KEY; key.offset = 0ull; while (1) { u64 dev_extent_len; ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); if (ret < 0) break; if (ret > 0) { if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { ret = btrfs_next_leaf(root, path); if (ret < 0) break; if (ret > 0) { ret = 0; break; } } else { ret = 0; } } l = path->nodes[0]; slot = path->slots[0]; btrfs_item_key_to_cpu(l, &found_key, slot); if (found_key.objectid != scrub_dev->devid) break; if (found_key.type != BTRFS_DEV_EXTENT_KEY) break; if (found_key.offset >= end) break; if (found_key.offset < key.offset) break; dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent); dev_extent_len = btrfs_dev_extent_length(l, dev_extent); if (found_key.offset + dev_extent_len <= start) goto skip; chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent); /* * get a reference on the corresponding block group to prevent * the chunk from going away while we scrub it */ cache = btrfs_lookup_block_group(fs_info, chunk_offset); /* some chunks are removed but not committed to disk yet, * continue scrubbing */ if (!cache) goto skip; ASSERT(cache->start <= chunk_offset); /* * We are using the commit root to search for device extents, so * that means we could have found a device extent item from a * block group that was deleted in the current transaction. The * logical start offset of the deleted block group, stored at * @chunk_offset, might be part of the logical address range of * a new block group (which uses different physical extents). * In this case btrfs_lookup_block_group() has returned the new * block group, and its start address is less than @chunk_offset. * * We skip such new block groups, because it's pointless to * process them, as we won't find their extents because we search * for them using the commit root of the extent tree. For a device * replace it's also fine to skip it, we won't miss copying them * to the target device because we have the write duplication * setup through the regular write path (by btrfs_map_block()), * and we have committed a transaction when we started the device * replace, right after setting up the device replace state. */ if (cache->start < chunk_offset) { btrfs_put_block_group(cache); goto skip; } if (sctx->is_dev_replace && btrfs_is_zoned(fs_info)) { if (!test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags)) { btrfs_put_block_group(cache); goto skip; } } /* * Make sure that while we are scrubbing the corresponding block * group doesn't get its logical address and its device extents * reused for another block group, which can possibly be of a * different type and different profile. We do this to prevent * false error detections and crashes due to bogus attempts to * repair extents. */ spin_lock(&cache->lock); if (test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags)) { spin_unlock(&cache->lock); btrfs_put_block_group(cache); goto skip; } btrfs_freeze_block_group(cache); spin_unlock(&cache->lock); /* * we need call btrfs_inc_block_group_ro() with scrubs_paused, * to avoid deadlock caused by: * btrfs_inc_block_group_ro() * -> btrfs_wait_for_commit() * -> btrfs_commit_transaction() * -> btrfs_scrub_pause() */ scrub_pause_on(fs_info); /* * Don't do chunk preallocation for scrub. * * This is especially important for SYSTEM bgs, or we can hit * -EFBIG from btrfs_finish_chunk_alloc() like: * 1. The only SYSTEM bg is marked RO. * Since SYSTEM bg is small, that's pretty common. * 2. New SYSTEM bg will be allocated * Due to regular version will allocate new chunk. * 3. New SYSTEM bg is empty and will get cleaned up * Before cleanup really happens, it's marked RO again. * 4. Empty SYSTEM bg get scrubbed * We go back to 2. * * This can easily boost the amount of SYSTEM chunks if cleaner * thread can't be triggered fast enough, and use up all space * of btrfs_super_block::sys_chunk_array * * While for dev replace, we need to try our best to mark block * group RO, to prevent race between: * - Write duplication * Contains latest data * - Scrub copy * Contains data from commit tree * * If target block group is not marked RO, nocow writes can * be overwritten by scrub copy, causing data corruption. * So for dev-replace, it's not allowed to continue if a block * group is not RO. */ ret = btrfs_inc_block_group_ro(cache, sctx->is_dev_replace); if (!ret && sctx->is_dev_replace) { ret = finish_extent_writes_for_zoned(root, cache); if (ret) { btrfs_dec_block_group_ro(cache); scrub_pause_off(fs_info); btrfs_put_block_group(cache); break; } } if (ret == 0) { ro_set = 1; } else if (ret == -ENOSPC && !sctx->is_dev_replace && !(cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK)) { /* * btrfs_inc_block_group_ro return -ENOSPC when it * failed in creating new chunk for metadata. * It is not a problem for scrub, because * metadata are always cowed, and our scrub paused * commit_transactions. * * For RAID56 chunks, we have to mark them read-only * for scrub, as later we would use our own cache * out of RAID56 realm. * Thus we want the RAID56 bg to be marked RO to * prevent RMW from screwing up out cache. */ ro_set = 0; } else if (ret == -ETXTBSY) { btrfs_warn(fs_info, "scrub: skipping scrub of block group %llu due to active swapfile", cache->start); scrub_pause_off(fs_info); ret = 0; goto skip_unfreeze; } else { btrfs_warn(fs_info, "scrub: failed setting block group ro: %d", ret); btrfs_unfreeze_block_group(cache); btrfs_put_block_group(cache); scrub_pause_off(fs_info); break; } /* * Now the target block is marked RO, wait for nocow writes to * finish before dev-replace. * COW is fine, as COW never overwrites extents in commit tree. */ if (sctx->is_dev_replace) { btrfs_wait_nocow_writers(cache); btrfs_wait_ordered_roots(fs_info, U64_MAX, cache); } scrub_pause_off(fs_info); down_write(&dev_replace->rwsem); dev_replace->cursor_right = found_key.offset + dev_extent_len; dev_replace->cursor_left = found_key.offset; dev_replace->item_needs_writeback = 1; up_write(&dev_replace->rwsem); ret = scrub_chunk(sctx, cache, scrub_dev, found_key.offset, dev_extent_len); if (sctx->is_dev_replace && !btrfs_finish_block_group_to_copy(dev_replace->srcdev, cache, found_key.offset)) ro_set = 0; down_write(&dev_replace->rwsem); dev_replace->cursor_left = dev_replace->cursor_right; dev_replace->item_needs_writeback = 1; up_write(&dev_replace->rwsem); if (ro_set) btrfs_dec_block_group_ro(cache); /* * We might have prevented the cleaner kthread from deleting * this block group if it was already unused because we raced * and set it to RO mode first. So add it back to the unused * list, otherwise it might not ever be deleted unless a manual * balance is triggered or it becomes used and unused again. */ spin_lock(&cache->lock); if (!test_bit(BLOCK_GROUP_FLAG_REMOVED, &cache->runtime_flags) && !cache->ro && cache->reserved == 0 && cache->used == 0) { spin_unlock(&cache->lock); if (btrfs_test_opt(fs_info, DISCARD_ASYNC)) btrfs_discard_queue_work(&fs_info->discard_ctl, cache); else btrfs_mark_bg_unused(cache); } else { spin_unlock(&cache->lock); } skip_unfreeze: btrfs_unfreeze_block_group(cache); btrfs_put_block_group(cache); if (ret) break; if (unlikely(sctx->is_dev_replace && atomic64_read(&dev_replace->num_write_errors) > 0)) { ret = -EIO; break; } if (sctx->stat.malloc_errors > 0) { ret = -ENOMEM; break; } skip: key.offset = found_key.offset + dev_extent_len; btrfs_release_path(path); } return ret; } static int scrub_one_super(struct scrub_ctx *sctx, struct btrfs_device *dev, struct page *page, u64 physical, u64 generation) { struct btrfs_fs_info *fs_info = sctx->fs_info; struct btrfs_super_block *sb = page_address(page); int ret; ret = bdev_rw_virt(dev->bdev, physical >> SECTOR_SHIFT, sb, BTRFS_SUPER_INFO_SIZE, REQ_OP_READ); if (ret < 0) return ret; ret = btrfs_check_super_csum(fs_info, sb); if (unlikely(ret != 0)) { btrfs_err_rl(fs_info, "scrub: super block at physical %llu devid %llu has bad csum", physical, dev->devid); return -EIO; } if (unlikely(btrfs_super_generation(sb) != generation)) { btrfs_err_rl(fs_info, "scrub: super block at physical %llu devid %llu has bad generation %llu expect %llu", physical, dev->devid, btrfs_super_generation(sb), generation); return -EUCLEAN; } return btrfs_validate_super(fs_info, sb, -1); } static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx, struct btrfs_device *scrub_dev) { int i; u64 bytenr; u64 gen; int ret = 0; struct page *page; struct btrfs_fs_info *fs_info = sctx->fs_info; if (unlikely(BTRFS_FS_ERROR(fs_info))) return -EROFS; page = alloc_page(GFP_KERNEL); if (!page) { spin_lock(&sctx->stat_lock); sctx->stat.malloc_errors++; spin_unlock(&sctx->stat_lock); return -ENOMEM; } /* Seed devices of a new filesystem has their own generation. */ if (scrub_dev->fs_devices != fs_info->fs_devices) gen = scrub_dev->generation; else gen = btrfs_get_last_trans_committed(fs_info); for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { ret = btrfs_sb_log_location(scrub_dev, i, 0, &bytenr); if (ret == -ENOENT) break; if (ret) { spin_lock(&sctx->stat_lock); sctx->stat.super_errors++; spin_unlock(&sctx->stat_lock); continue; } if (bytenr + BTRFS_SUPER_INFO_SIZE > scrub_dev->commit_total_bytes) break; if (!btrfs_check_super_location(scrub_dev, bytenr)) continue; ret = scrub_one_super(sctx, scrub_dev, page, bytenr, gen); if (ret) { spin_lock(&sctx->stat_lock); sctx->stat.super_errors++; spin_unlock(&sctx->stat_lock); } } __free_page(page); return 0; } static void scrub_workers_put(struct btrfs_fs_info *fs_info) { if (refcount_dec_and_mutex_lock(&fs_info->scrub_workers_refcnt, &fs_info->scrub_lock)) { struct workqueue_struct *scrub_workers = fs_info->scrub_workers; fs_info->scrub_workers = NULL; mutex_unlock(&fs_info->scrub_lock); if (scrub_workers) destroy_workqueue(scrub_workers); } } /* * get a reference count on fs_info->scrub_workers. start worker if necessary */ static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info) { struct workqueue_struct *scrub_workers = NULL; unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND; int max_active = fs_info->thread_pool_size; int ret = -ENOMEM; if (refcount_inc_not_zero(&fs_info->scrub_workers_refcnt)) return 0; scrub_workers = alloc_workqueue("btrfs-scrub", flags, max_active); if (!scrub_workers) return -ENOMEM; mutex_lock(&fs_info->scrub_lock); if (refcount_read(&fs_info->scrub_workers_refcnt) == 0) { ASSERT(fs_info->scrub_workers == NULL); fs_info->scrub_workers = scrub_workers; refcount_set(&fs_info->scrub_workers_refcnt, 1); mutex_unlock(&fs_info->scrub_lock); return 0; } /* Other thread raced in and created the workers for us */ refcount_inc(&fs_info->scrub_workers_refcnt); mutex_unlock(&fs_info->scrub_lock); ret = 0; destroy_workqueue(scrub_workers); return ret; } int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start, u64 end, struct btrfs_scrub_progress *progress, bool readonly, bool is_dev_replace) { struct btrfs_dev_lookup_args args = { .devid = devid }; struct scrub_ctx *sctx; int ret; struct btrfs_device *dev; unsigned int nofs_flag; bool need_commit = false; /* Set the basic fallback @last_physical before we got a sctx. */ if (progress) progress->last_physical = start; if (btrfs_fs_closing(fs_info)) return -EAGAIN; /* At mount time we have ensured nodesize is in the range of [4K, 64K]. */ ASSERT(fs_info->nodesize <= BTRFS_STRIPE_LEN); /* * SCRUB_MAX_SECTORS_PER_BLOCK is calculated using the largest possible * value (max nodesize / min sectorsize), thus nodesize should always * be fine. */ ASSERT(fs_info->nodesize <= SCRUB_MAX_SECTORS_PER_BLOCK << fs_info->sectorsize_bits); /* Allocate outside of device_list_mutex */ sctx = scrub_setup_ctx(fs_info, is_dev_replace); if (IS_ERR(sctx)) return PTR_ERR(sctx); sctx->stat.last_physical = start; ret = scrub_workers_get(fs_info); if (ret) goto out_free_ctx; mutex_lock(&fs_info->fs_devices->device_list_mutex); dev = btrfs_find_device(fs_info->fs_devices, &args); if (!dev || (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) && !is_dev_replace)) { mutex_unlock(&fs_info->fs_devices->device_list_mutex); ret = -ENODEV; goto out; } if (!is_dev_replace && !readonly && !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) { mutex_unlock(&fs_info->fs_devices->device_list_mutex); btrfs_err(fs_info, "scrub: devid %llu: filesystem on %s is not writable", devid, btrfs_dev_name(dev)); ret = -EROFS; goto out; } mutex_lock(&fs_info->scrub_lock); if (unlikely(!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) || test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &dev->dev_state))) { mutex_unlock(&fs_info->scrub_lock); mutex_unlock(&fs_info->fs_devices->device_list_mutex); ret = -EIO; goto out; } down_read(&fs_info->dev_replace.rwsem); if (dev->scrub_ctx || (!is_dev_replace && btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) { up_read(&fs_info->dev_replace.rwsem); mutex_unlock(&fs_info->scrub_lock); mutex_unlock(&fs_info->fs_devices->device_list_mutex); ret = -EINPROGRESS; goto out; } up_read(&fs_info->dev_replace.rwsem); sctx->readonly = readonly; dev->scrub_ctx = sctx; mutex_unlock(&fs_info->fs_devices->device_list_mutex); /* * checking @scrub_pause_req here, we can avoid * race between committing transaction and scrubbing. */ __scrub_blocked_if_needed(fs_info); atomic_inc(&fs_info->scrubs_running); mutex_unlock(&fs_info->scrub_lock); /* * In order to avoid deadlock with reclaim when there is a transaction * trying to pause scrub, make sure we use GFP_NOFS for all the * allocations done at btrfs_scrub_sectors() and scrub_sectors_for_parity() * invoked by our callees. The pausing request is done when the * transaction commit starts, and it blocks the transaction until scrub * is paused (done at specific points at scrub_stripe() or right above * before incrementing fs_info->scrubs_running). */ nofs_flag = memalloc_nofs_save(); if (!is_dev_replace) { u64 old_super_errors; spin_lock(&sctx->stat_lock); old_super_errors = sctx->stat.super_errors; spin_unlock(&sctx->stat_lock); btrfs_info(fs_info, "scrub: started on devid %llu", devid); /* * by holding device list mutex, we can * kick off writing super in log tree sync. */ mutex_lock(&fs_info->fs_devices->device_list_mutex); ret = scrub_supers(sctx, dev); mutex_unlock(&fs_info->fs_devices->device_list_mutex); spin_lock(&sctx->stat_lock); /* * Super block errors found, but we can not commit transaction * at current context, since btrfs_commit_transaction() needs * to pause the current running scrub (hold by ourselves). */ if (sctx->stat.super_errors > old_super_errors && !sctx->readonly) need_commit = true; spin_unlock(&sctx->stat_lock); } if (!ret) ret = scrub_enumerate_chunks(sctx, dev, start, end); memalloc_nofs_restore(nofs_flag); atomic_dec(&fs_info->scrubs_running); wake_up(&fs_info->scrub_pause_wait); if (progress) memcpy(progress, &sctx->stat, sizeof(*progress)); if (!is_dev_replace) btrfs_info(fs_info, "scrub: %s on devid %llu with status: %d", ret ? "not finished" : "finished", devid, ret); mutex_lock(&fs_info->scrub_lock); dev->scrub_ctx = NULL; mutex_unlock(&fs_info->scrub_lock); scrub_workers_put(fs_info); scrub_put_ctx(sctx); /* * We found some super block errors before, now try to force a * transaction commit, as scrub has finished. */ if (need_commit) { struct btrfs_trans_handle *trans; trans = btrfs_start_transaction(fs_info->tree_root, 0); if (IS_ERR(trans)) { ret = PTR_ERR(trans); btrfs_err(fs_info, "scrub: failed to start transaction to fix super block errors: %d", ret); return ret; } ret = btrfs_commit_transaction(trans); if (ret < 0) btrfs_err(fs_info, "scrub: failed to commit transaction to fix super block errors: %d", ret); } return ret; out: scrub_workers_put(fs_info); out_free_ctx: scrub_free_ctx(sctx); return ret; } void btrfs_scrub_pause(struct btrfs_fs_info *fs_info) { mutex_lock(&fs_info->scrub_lock); atomic_inc(&fs_info->scrub_pause_req); while (atomic_read(&fs_info->scrubs_paused) != atomic_read(&fs_info->scrubs_running)) { mutex_unlock(&fs_info->scrub_lock); wait_event(fs_info->scrub_pause_wait, atomic_read(&fs_info->scrubs_paused) == atomic_read(&fs_info->scrubs_running)); mutex_lock(&fs_info->scrub_lock); } mutex_unlock(&fs_info->scrub_lock); } void btrfs_scrub_continue(struct btrfs_fs_info *fs_info) { atomic_dec(&fs_info->scrub_pause_req); wake_up(&fs_info->scrub_pause_wait); } int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info) { mutex_lock(&fs_info->scrub_lock); if (!atomic_read(&fs_info->scrubs_running)) { mutex_unlock(&fs_info->scrub_lock); return -ENOTCONN; } atomic_inc(&fs_info->scrub_cancel_req); while (atomic_read(&fs_info->scrubs_running)) { mutex_unlock(&fs_info->scrub_lock); wait_event(fs_info->scrub_pause_wait, atomic_read(&fs_info->scrubs_running) == 0); mutex_lock(&fs_info->scrub_lock); } atomic_dec(&fs_info->scrub_cancel_req); mutex_unlock(&fs_info->scrub_lock); return 0; } int btrfs_scrub_cancel_dev(struct btrfs_device *dev) { struct btrfs_fs_info *fs_info = dev->fs_info; struct scrub_ctx *sctx; mutex_lock(&fs_info->scrub_lock); sctx = dev->scrub_ctx; if (!sctx) { mutex_unlock(&fs_info->scrub_lock); return -ENOTCONN; } atomic_inc(&sctx->cancel_req); while (dev->scrub_ctx) { mutex_unlock(&fs_info->scrub_lock); wait_event(fs_info->scrub_pause_wait, dev->scrub_ctx == NULL); mutex_lock(&fs_info->scrub_lock); } mutex_unlock(&fs_info->scrub_lock); return 0; } int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid, struct btrfs_scrub_progress *progress) { struct btrfs_dev_lookup_args args = { .devid = devid }; struct btrfs_device *dev; struct scrub_ctx *sctx = NULL; mutex_lock(&fs_info->fs_devices->device_list_mutex); dev = btrfs_find_device(fs_info->fs_devices, &args); if (dev) sctx = dev->scrub_ctx; if (sctx) memcpy(progress, &sctx->stat, sizeof(*progress)); mutex_unlock(&fs_info->fs_devices->device_list_mutex); return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV; } |
| 3 3 3 3 3 3 3 3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 | // SPDX-License-Identifier: GPL-2.0+ /* * HID driver for UC-Logic devices not fully compliant with HID standard * - original and fixed report descriptors * * Copyright (c) 2010-2017 Nikolai Kondrashov * Copyright (c) 2013 Martin Rusko */ /* * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation; either version 2 of the License, or (at your option) * any later version. */ #include "hid-uclogic-rdesc.h" #include <linux/slab.h> #include <linux/unaligned.h> #include <kunit/visibility.h> /* Fixed WP4030U report descriptor */ const __u8 uclogic_rdesc_wp4030u_fixed_arr[] = { 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x01, /* Usage (Digitizer), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x09, /* Report ID (9), */ 0x09, 0x20, /* Usage (Stylus), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x09, 0x42, /* Usage (Tip Switch), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x09, 0x46, /* Usage (Tablet Pick), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x05, /* Report Count (5), */ 0x81, 0x01, /* Input (Constant), */ 0x75, 0x10, /* Report Size (16), */ 0x95, 0x01, /* Report Count (1), */ 0x14, /* Logical Minimum (0), */ 0xA4, /* Push, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x55, 0xFD, /* Unit Exponent (-3), */ 0x65, 0x13, /* Unit (Inch), */ 0x34, /* Physical Minimum (0), */ 0x09, 0x30, /* Usage (X), */ 0x46, 0xA0, 0x0F, /* Physical Maximum (4000), */ 0x26, 0xFF, 0x7F, /* Logical Maximum (32767), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x31, /* Usage (Y), */ 0x46, 0xB8, 0x0B, /* Physical Maximum (3000), */ 0x26, 0xFF, 0x7F, /* Logical Maximum (32767), */ 0x81, 0x02, /* Input (Variable), */ 0xB4, /* Pop, */ 0x09, 0x30, /* Usage (Tip Pressure), */ 0x26, 0xFF, 0x03, /* Logical Maximum (1023), */ 0x81, 0x02, /* Input (Variable), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_wp4030u_fixed_size = sizeof(uclogic_rdesc_wp4030u_fixed_arr); /* Fixed WP5540U report descriptor */ const __u8 uclogic_rdesc_wp5540u_fixed_arr[] = { 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x01, /* Usage (Digitizer), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x09, /* Report ID (9), */ 0x09, 0x20, /* Usage (Stylus), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x09, 0x42, /* Usage (Tip Switch), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x09, 0x46, /* Usage (Tablet Pick), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x05, /* Report Count (5), */ 0x81, 0x01, /* Input (Constant), */ 0x75, 0x10, /* Report Size (16), */ 0x95, 0x01, /* Report Count (1), */ 0x14, /* Logical Minimum (0), */ 0xA4, /* Push, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x55, 0xFD, /* Unit Exponent (-3), */ 0x65, 0x13, /* Unit (Inch), */ 0x34, /* Physical Minimum (0), */ 0x09, 0x30, /* Usage (X), */ 0x46, 0x7C, 0x15, /* Physical Maximum (5500), */ 0x26, 0xFF, 0x7F, /* Logical Maximum (32767), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x31, /* Usage (Y), */ 0x46, 0xA0, 0x0F, /* Physical Maximum (4000), */ 0x26, 0xFF, 0x7F, /* Logical Maximum (32767), */ 0x81, 0x02, /* Input (Variable), */ 0xB4, /* Pop, */ 0x09, 0x30, /* Usage (Tip Pressure), */ 0x26, 0xFF, 0x03, /* Logical Maximum (1023), */ 0x81, 0x02, /* Input (Variable), */ 0xC0, /* End Collection, */ 0xC0, /* End Collection, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x02, /* Usage (Mouse), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x08, /* Report ID (8), */ 0x09, 0x01, /* Usage (Pointer), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x05, 0x09, /* Usage Page (Button), */ 0x19, 0x01, /* Usage Minimum (01h), */ 0x29, 0x03, /* Usage Maximum (03h), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x05, /* Report Count (5), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x75, 0x08, /* Report Size (8), */ 0x09, 0x30, /* Usage (X), */ 0x09, 0x31, /* Usage (Y), */ 0x15, 0x81, /* Logical Minimum (-127), */ 0x25, 0x7F, /* Logical Maximum (127), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x09, 0x38, /* Usage (Wheel), */ 0x15, 0xFF, /* Logical Minimum (-1), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x81, 0x01, /* Input (Constant), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_wp5540u_fixed_size = sizeof(uclogic_rdesc_wp5540u_fixed_arr); /* Fixed WP8060U report descriptor */ const __u8 uclogic_rdesc_wp8060u_fixed_arr[] = { 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x01, /* Usage (Digitizer), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x09, /* Report ID (9), */ 0x09, 0x20, /* Usage (Stylus), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x09, 0x42, /* Usage (Tip Switch), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x09, 0x46, /* Usage (Tablet Pick), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x05, /* Report Count (5), */ 0x81, 0x01, /* Input (Constant), */ 0x75, 0x10, /* Report Size (16), */ 0x95, 0x01, /* Report Count (1), */ 0x14, /* Logical Minimum (0), */ 0xA4, /* Push, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x55, 0xFD, /* Unit Exponent (-3), */ 0x65, 0x13, /* Unit (Inch), */ 0x34, /* Physical Minimum (0), */ 0x09, 0x30, /* Usage (X), */ 0x46, 0x40, 0x1F, /* Physical Maximum (8000), */ 0x26, 0xFF, 0x7F, /* Logical Maximum (32767), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x31, /* Usage (Y), */ 0x46, 0x70, 0x17, /* Physical Maximum (6000), */ 0x26, 0xFF, 0x7F, /* Logical Maximum (32767), */ 0x81, 0x02, /* Input (Variable), */ 0xB4, /* Pop, */ 0x09, 0x30, /* Usage (Tip Pressure), */ 0x26, 0xFF, 0x03, /* Logical Maximum (1023), */ 0x81, 0x02, /* Input (Variable), */ 0xC0, /* End Collection, */ 0xC0, /* End Collection, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x02, /* Usage (Mouse), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x08, /* Report ID (8), */ 0x09, 0x01, /* Usage (Pointer), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x05, 0x09, /* Usage Page (Button), */ 0x19, 0x01, /* Usage Minimum (01h), */ 0x29, 0x03, /* Usage Maximum (03h), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x05, /* Report Count (5), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x75, 0x08, /* Report Size (8), */ 0x09, 0x30, /* Usage (X), */ 0x09, 0x31, /* Usage (Y), */ 0x15, 0x81, /* Logical Minimum (-127), */ 0x25, 0x7F, /* Logical Maximum (127), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x09, 0x38, /* Usage (Wheel), */ 0x15, 0xFF, /* Logical Minimum (-1), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x81, 0x01, /* Input (Constant), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_wp8060u_fixed_size = sizeof(uclogic_rdesc_wp8060u_fixed_arr); /* Fixed WP1062 report descriptor */ const __u8 uclogic_rdesc_wp1062_fixed_arr[] = { 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x01, /* Usage (Digitizer), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x09, /* Report ID (9), */ 0x09, 0x20, /* Usage (Stylus), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x09, 0x42, /* Usage (Tip Switch), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x09, 0x46, /* Usage (Tablet Pick), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x04, /* Report Count (4), */ 0x81, 0x01, /* Input (Constant), */ 0x09, 0x32, /* Usage (In Range), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x75, 0x10, /* Report Size (16), */ 0x95, 0x01, /* Report Count (1), */ 0x14, /* Logical Minimum (0), */ 0xA4, /* Push, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x55, 0xFD, /* Unit Exponent (-3), */ 0x65, 0x13, /* Unit (Inch), */ 0x34, /* Physical Minimum (0), */ 0x09, 0x30, /* Usage (X), */ 0x46, 0x10, 0x27, /* Physical Maximum (10000), */ 0x26, 0x20, 0x4E, /* Logical Maximum (20000), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x31, /* Usage (Y), */ 0x46, 0xB7, 0x19, /* Physical Maximum (6583), */ 0x26, 0x6E, 0x33, /* Logical Maximum (13166), */ 0x81, 0x02, /* Input (Variable), */ 0xB4, /* Pop, */ 0x09, 0x30, /* Usage (Tip Pressure), */ 0x26, 0xFF, 0x03, /* Logical Maximum (1023), */ 0x81, 0x02, /* Input (Variable), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_wp1062_fixed_size = sizeof(uclogic_rdesc_wp1062_fixed_arr); /* Fixed PF1209 report descriptor */ const __u8 uclogic_rdesc_pf1209_fixed_arr[] = { 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x01, /* Usage (Digitizer), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x09, /* Report ID (9), */ 0x09, 0x20, /* Usage (Stylus), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x09, 0x42, /* Usage (Tip Switch), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x09, 0x46, /* Usage (Tablet Pick), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x05, /* Report Count (5), */ 0x81, 0x01, /* Input (Constant), */ 0x75, 0x10, /* Report Size (16), */ 0x95, 0x01, /* Report Count (1), */ 0x14, /* Logical Minimum (0), */ 0xA4, /* Push, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x55, 0xFD, /* Unit Exponent (-3), */ 0x65, 0x13, /* Unit (Inch), */ 0x34, /* Physical Minimum (0), */ 0x09, 0x30, /* Usage (X), */ 0x46, 0xE0, 0x2E, /* Physical Maximum (12000), */ 0x26, 0xFF, 0x7F, /* Logical Maximum (32767), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x31, /* Usage (Y), */ 0x46, 0x28, 0x23, /* Physical Maximum (9000), */ 0x26, 0xFF, 0x7F, /* Logical Maximum (32767), */ 0x81, 0x02, /* Input (Variable), */ 0xB4, /* Pop, */ 0x09, 0x30, /* Usage (Tip Pressure), */ 0x26, 0xFF, 0x03, /* Logical Maximum (1023), */ 0x81, 0x02, /* Input (Variable), */ 0xC0, /* End Collection, */ 0xC0, /* End Collection, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x02, /* Usage (Mouse), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x08, /* Report ID (8), */ 0x09, 0x01, /* Usage (Pointer), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x05, 0x09, /* Usage Page (Button), */ 0x19, 0x01, /* Usage Minimum (01h), */ 0x29, 0x03, /* Usage Maximum (03h), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x05, /* Report Count (5), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x75, 0x08, /* Report Size (8), */ 0x09, 0x30, /* Usage (X), */ 0x09, 0x31, /* Usage (Y), */ 0x15, 0x81, /* Logical Minimum (-127), */ 0x25, 0x7F, /* Logical Maximum (127), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x09, 0x38, /* Usage (Wheel), */ 0x15, 0xFF, /* Logical Minimum (-1), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x81, 0x01, /* Input (Constant), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_pf1209_fixed_size = sizeof(uclogic_rdesc_pf1209_fixed_arr); /* Fixed PID 0522 tablet report descriptor, interface 0 (stylus) */ const __u8 uclogic_rdesc_twhl850_fixed0_arr[] = { 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x01, /* Usage (Digitizer), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x09, /* Report ID (9), */ 0x09, 0x20, /* Usage (Stylus), */ 0xA0, /* Collection (Physical), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x03, /* Report Count (3), */ 0x09, 0x42, /* Usage (Tip Switch), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x09, 0x46, /* Usage (Tablet Pick), */ 0x81, 0x02, /* Input (Variable), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x95, 0x01, /* Report Count (1), */ 0x09, 0x32, /* Usage (In Range), */ 0x81, 0x02, /* Input (Variable), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x75, 0x10, /* Report Size (16), */ 0xA4, /* Push, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x65, 0x13, /* Unit (Inch), */ 0x55, 0xFD, /* Unit Exponent (-3), */ 0x34, /* Physical Minimum (0), */ 0x09, 0x30, /* Usage (X), */ 0x46, 0x40, 0x1F, /* Physical Maximum (8000), */ 0x26, 0x00, 0x7D, /* Logical Maximum (32000), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x31, /* Usage (Y), */ 0x46, 0x88, 0x13, /* Physical Maximum (5000), */ 0x26, 0x20, 0x4E, /* Logical Maximum (20000), */ 0x81, 0x02, /* Input (Variable), */ 0xB4, /* Pop, */ 0x09, 0x30, /* Usage (Tip Pressure), */ 0x26, 0xFF, 0x03, /* Logical Maximum (1023), */ 0x81, 0x02, /* Input (Variable), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_twhl850_fixed0_size = sizeof(uclogic_rdesc_twhl850_fixed0_arr); /* Fixed PID 0522 tablet report descriptor, interface 1 (mouse) */ const __u8 uclogic_rdesc_twhl850_fixed1_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x02, /* Usage (Mouse), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x01, /* Report ID (1), */ 0x09, 0x01, /* Usage (Pointer), */ 0xA0, /* Collection (Physical), */ 0x05, 0x09, /* Usage Page (Button), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x03, /* Report Count (3), */ 0x19, 0x01, /* Usage Minimum (01h), */ 0x29, 0x03, /* Usage Maximum (03h), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x05, /* Report Count (5), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x30, /* Usage (X), */ 0x09, 0x31, /* Usage (Y), */ 0x16, 0x00, 0x80, /* Logical Minimum (-32768), */ 0x26, 0xFF, 0x7F, /* Logical Maximum (32767), */ 0x75, 0x10, /* Report Size (16), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x09, 0x38, /* Usage (Wheel), */ 0x15, 0xFF, /* Logical Minimum (-1), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x01, /* Report Count (1), */ 0x75, 0x08, /* Report Size (8), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_twhl850_fixed1_size = sizeof(uclogic_rdesc_twhl850_fixed1_arr); /* Fixed PID 0522 tablet report descriptor, interface 2 (frame buttons) */ const __u8 uclogic_rdesc_twhl850_fixed2_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x06, /* Usage (Keyboard), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x03, /* Report ID (3), */ 0x05, 0x07, /* Usage Page (Keyboard), */ 0x14, /* Logical Minimum (0), */ 0x19, 0xE0, /* Usage Minimum (KB Leftcontrol), */ 0x29, 0xE7, /* Usage Maximum (KB Right GUI), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x08, /* Report Count (8), */ 0x81, 0x02, /* Input (Variable), */ 0x18, /* Usage Minimum (None), */ 0x29, 0xFF, /* Usage Maximum (FFh), */ 0x26, 0xFF, 0x00, /* Logical Maximum (255), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x06, /* Report Count (6), */ 0x80, /* Input, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_twhl850_fixed2_size = sizeof(uclogic_rdesc_twhl850_fixed2_arr); /* Fixed TWHA60 report descriptor, interface 0 (stylus) */ const __u8 uclogic_rdesc_twha60_fixed0_arr[] = { 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x01, /* Usage (Digitizer), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x09, /* Report ID (9), */ 0x09, 0x20, /* Usage (Stylus), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x09, 0x42, /* Usage (Tip Switch), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x09, 0x46, /* Usage (Tablet Pick), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x04, /* Report Count (4), */ 0x81, 0x01, /* Input (Constant), */ 0x09, 0x32, /* Usage (In Range), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x75, 0x10, /* Report Size (16), */ 0x95, 0x01, /* Report Count (1), */ 0x14, /* Logical Minimum (0), */ 0xA4, /* Push, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x55, 0xFD, /* Unit Exponent (-3), */ 0x65, 0x13, /* Unit (Inch), */ 0x34, /* Physical Minimum (0), */ 0x09, 0x30, /* Usage (X), */ 0x46, 0x10, 0x27, /* Physical Maximum (10000), */ 0x27, 0x3F, 0x9C, 0x00, 0x00, /* Logical Maximum (39999), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x31, /* Usage (Y), */ 0x46, 0x6A, 0x18, /* Physical Maximum (6250), */ 0x26, 0xA7, 0x61, /* Logical Maximum (24999), */ 0x81, 0x02, /* Input (Variable), */ 0xB4, /* Pop, */ 0x09, 0x30, /* Usage (Tip Pressure), */ 0x26, 0xFF, 0x03, /* Logical Maximum (1023), */ 0x81, 0x02, /* Input (Variable), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_twha60_fixed0_size = sizeof(uclogic_rdesc_twha60_fixed0_arr); /* Fixed TWHA60 report descriptor, interface 1 (frame buttons) */ const __u8 uclogic_rdesc_twha60_fixed1_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x06, /* Usage (Keyboard), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x05, /* Report ID (5), */ 0x05, 0x07, /* Usage Page (Keyboard), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x08, /* Report Count (8), */ 0x81, 0x01, /* Input (Constant), */ 0x95, 0x0C, /* Report Count (12), */ 0x19, 0x3A, /* Usage Minimum (KB F1), */ 0x29, 0x45, /* Usage Maximum (KB F12), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x0C, /* Report Count (12), */ 0x19, 0x68, /* Usage Minimum (KB F13), */ 0x29, 0x73, /* Usage Maximum (KB F24), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x08, /* Report Count (8), */ 0x81, 0x01, /* Input (Constant), */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_twha60_fixed1_size = sizeof(uclogic_rdesc_twha60_fixed1_arr); /* Fixed report descriptor template for (tweaked) v1 pen reports */ const __u8 uclogic_rdesc_v1_pen_template_arr[] = { 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x01, /* Usage (Digitizer), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x07, /* Report ID (7), */ 0x09, 0x20, /* Usage (Stylus), */ 0xA0, /* Collection (Physical), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x09, 0x42, /* Usage (Tip Switch), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x09, 0x46, /* Usage (Tablet Pick), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x09, 0x32, /* Usage (In Range), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x75, 0x10, /* Report Size (16), */ 0x95, 0x01, /* Report Count (1), */ 0xA4, /* Push, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x65, 0x13, /* Unit (Inch), */ 0x55, 0xFD, /* Unit Exponent (-3), */ 0x34, /* Physical Minimum (0), */ 0x09, 0x30, /* Usage (X), */ 0x27, UCLOGIC_RDESC_PEN_PH(X_LM), /* Logical Maximum (PLACEHOLDER), */ 0x47, UCLOGIC_RDESC_PEN_PH(X_PM), /* Physical Maximum (PLACEHOLDER), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x31, /* Usage (Y), */ 0x27, UCLOGIC_RDESC_PEN_PH(Y_LM), /* Logical Maximum (PLACEHOLDER), */ 0x47, UCLOGIC_RDESC_PEN_PH(Y_PM), /* Physical Maximum (PLACEHOLDER), */ 0x81, 0x02, /* Input (Variable), */ 0xB4, /* Pop, */ 0x09, 0x30, /* Usage (Tip Pressure), */ 0x27, UCLOGIC_RDESC_PEN_PH(PRESSURE_LM), /* Logical Maximum (PLACEHOLDER), */ 0x81, 0x02, /* Input (Variable), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_v1_pen_template_size = sizeof(uclogic_rdesc_v1_pen_template_arr); /* Fixed report descriptor template for (tweaked) v2 pen reports */ const __u8 uclogic_rdesc_v2_pen_template_arr[] = { 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x01, /* Usage (Digitizer), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x08, /* Report ID (8), */ 0x09, 0x20, /* Usage (Stylus), */ 0xA0, /* Collection (Physical), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x09, 0x42, /* Usage (Tip Switch), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x09, 0x46, /* Usage (Tablet Pick), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x09, 0x32, /* Usage (In Range), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x95, 0x01, /* Report Count (1), */ 0xA4, /* Push, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x65, 0x13, /* Unit (Inch), */ 0x55, 0xFD, /* Unit Exponent (-3), */ 0x75, 0x18, /* Report Size (24), */ 0x34, /* Physical Minimum (0), */ 0x09, 0x30, /* Usage (X), */ 0x27, UCLOGIC_RDESC_PEN_PH(X_LM), /* Logical Maximum (PLACEHOLDER), */ 0x47, UCLOGIC_RDESC_PEN_PH(X_PM), /* Physical Maximum (PLACEHOLDER), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x31, /* Usage (Y), */ 0x27, UCLOGIC_RDESC_PEN_PH(Y_LM), /* Logical Maximum (PLACEHOLDER), */ 0x47, UCLOGIC_RDESC_PEN_PH(Y_PM), /* Physical Maximum (PLACEHOLDER), */ 0x81, 0x02, /* Input (Variable), */ 0xB4, /* Pop, */ 0x09, 0x30, /* Usage (Tip Pressure), */ 0x75, 0x10, /* Report Size (16), */ 0x27, UCLOGIC_RDESC_PEN_PH(PRESSURE_LM), /* Logical Maximum (PLACEHOLDER), */ 0x81, 0x02, /* Input (Variable), */ 0x54, /* Unit Exponent (0), */ 0x65, 0x14, /* Unit (Degrees), */ 0x35, 0xC4, /* Physical Minimum (-60), */ 0x45, 0x3C, /* Physical Maximum (60), */ 0x15, 0xC4, /* Logical Minimum (-60), */ 0x25, 0x3C, /* Logical Maximum (60), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x02, /* Report Count (2), */ 0x09, 0x3D, /* Usage (X Tilt), */ 0x09, 0x3E, /* Usage (Y Tilt), */ 0x81, 0x02, /* Input (Variable), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_v2_pen_template_size = sizeof(uclogic_rdesc_v2_pen_template_arr); /* * Expand to the contents of a generic frame buttons report descriptor. * * @_id: The report ID to use. * @_size: Size of the report to pad to, including report ID, bytes. */ #define UCLOGIC_RDESC_FRAME_BUTTONS_BYTES(_id, _size) \ 0x05, 0x01, /* Usage Page (Desktop), */ \ 0x09, 0x07, /* Usage (Keypad), */ \ 0xA1, 0x01, /* Collection (Application), */ \ 0x85, (_id), /* Report ID (_id), */ \ 0x14, /* Logical Minimum (0), */ \ 0x25, 0x01, /* Logical Maximum (1), */ \ 0x75, 0x01, /* Report Size (1), */ \ 0x05, 0x0D, /* Usage Page (Digitizer), */ \ 0x09, 0x39, /* Usage (Tablet Function Keys), */ \ 0xA0, /* Collection (Physical), */ \ 0x09, 0x44, /* Usage (Barrel Switch), */ \ 0x95, 0x01, /* Report Count (1), */ \ 0x81, 0x02, /* Input (Variable), */ \ 0x05, 0x01, /* Usage Page (Desktop), */ \ 0x09, 0x30, /* Usage (X), */ \ 0x09, 0x31, /* Usage (Y), */ \ 0x95, 0x02, /* Report Count (2), */ \ 0x81, 0x02, /* Input (Variable), */ \ 0x95, 0x15, /* Report Count (21), */ \ 0x81, 0x01, /* Input (Constant), */ \ 0x05, 0x09, /* Usage Page (Button), */ \ 0x19, 0x01, /* Usage Minimum (01h), */ \ 0x29, 0x0A, /* Usage Maximum (0Ah), */ \ 0x95, 0x0A, /* Report Count (10), */ \ 0x81, 0x02, /* Input (Variable), */ \ 0xC0, /* End Collection, */ \ 0x05, 0x01, /* Usage Page (Desktop), */ \ 0x09, 0x05, /* Usage (Gamepad), */ \ 0xA0, /* Collection (Physical), */ \ 0x05, 0x09, /* Usage Page (Button), */ \ 0x19, 0x01, /* Usage Minimum (01h), */ \ 0x29, 0x0A, /* Usage Maximum (0Ah), */ \ 0x95, 0x0A, /* Report Count (10), */ \ 0x81, 0x02, /* Input (Variable), */ \ 0x95, ((_size) * 8 - 52), \ /* Report Count (padding), */ \ 0x81, 0x01, /* Input (Constant), */ \ 0xC0, /* End Collection, */ \ 0xC0 /* End Collection */ /* Fixed report descriptor for (tweaked) v1 frame reports */ const __u8 uclogic_rdesc_v1_frame_arr[] = { UCLOGIC_RDESC_FRAME_BUTTONS_BYTES(UCLOGIC_RDESC_V1_FRAME_ID, 8) }; const size_t uclogic_rdesc_v1_frame_size = sizeof(uclogic_rdesc_v1_frame_arr); /* Fixed report descriptor for (tweaked) v2 frame button reports */ const __u8 uclogic_rdesc_v2_frame_buttons_arr[] = { UCLOGIC_RDESC_FRAME_BUTTONS_BYTES(UCLOGIC_RDESC_V2_FRAME_BUTTONS_ID, 12) }; const size_t uclogic_rdesc_v2_frame_buttons_size = sizeof(uclogic_rdesc_v2_frame_buttons_arr); /* Fixed report descriptor for (tweaked) v2 frame touch ring reports */ const __u8 uclogic_rdesc_v2_frame_touch_ring_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x07, /* Usage (Keypad), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, UCLOGIC_RDESC_V2_FRAME_TOUCH_ID, /* Report ID (TOUCH_ID), */ 0x14, /* Logical Minimum (0), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x39, /* Usage (Tablet Function Keys), */ 0xA0, /* Collection (Physical), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x05, 0x09, /* Usage Page (Button), */ 0x09, 0x01, /* Usage (01h), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x07, /* Report Count (7), */ 0x81, 0x01, /* Input (Constant), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x0A, 0xFF, 0xFF, /* Usage (FFFFh), */ 0x26, 0xFF, 0x00, /* Logical Maximum (255), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x38, /* Usage (Wheel), */ 0x95, 0x01, /* Report Count (1), */ 0x15, 0x00, /* Logical Minimum (0), */ 0x25, 0x0B, /* Logical Maximum (11), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x30, /* Usage (X), */ 0x09, 0x31, /* Usage (Y), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x2E, /* Report Count (46), */ 0x81, 0x01, /* Input (Constant), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_v2_frame_touch_ring_size = sizeof(uclogic_rdesc_v2_frame_touch_ring_arr); /* Fixed report descriptor for (tweaked) v2 frame touch strip reports */ const __u8 uclogic_rdesc_v2_frame_touch_strip_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x07, /* Usage (Keypad), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, UCLOGIC_RDESC_V2_FRAME_TOUCH_ID, /* Report ID (TOUCH_ID), */ 0x14, /* Logical Minimum (0), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x39, /* Usage (Tablet Function Keys), */ 0xA0, /* Collection (Physical), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x05, 0x09, /* Usage Page (Button), */ 0x09, 0x01, /* Usage (01h), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x07, /* Report Count (7), */ 0x81, 0x01, /* Input (Constant), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x0A, 0xFF, 0xFF, /* Usage (FFFFh), */ 0x26, 0xFF, 0x00, /* Logical Maximum (255), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x33, /* Usage (Rx), */ 0x09, 0x34, /* Usage (Ry), */ 0x95, 0x01, /* Report Count (1), */ 0x15, 0x00, /* Logical Minimum (0), */ 0x25, 0x07, /* Logical Maximum (7), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x30, /* Usage (X), */ 0x09, 0x31, /* Usage (Y), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x2E, /* Report Count (46), */ 0x81, 0x01, /* Input (Constant), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_v2_frame_touch_strip_size = sizeof(uclogic_rdesc_v2_frame_touch_strip_arr); /* Fixed report descriptor for (tweaked) v2 frame dial reports */ const __u8 uclogic_rdesc_v2_frame_dial_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x07, /* Usage (Keypad), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, UCLOGIC_RDESC_V2_FRAME_DIAL_ID, /* Report ID (DIAL_ID), */ 0x14, /* Logical Minimum (0), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x39, /* Usage (Tablet Function Keys), */ 0xA0, /* Collection (Physical), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x09, /* Usage Page (Button), */ 0x09, 0x01, /* Usage (01h), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x06, /* Report Count (6), */ 0x81, 0x01, /* Input (Constant), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x0A, 0xFF, 0xFF, /* Usage (FFFFh), */ 0x26, 0xFF, 0x00, /* Logical Maximum (255), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x38, /* Usage (Wheel), */ 0x95, 0x01, /* Report Count (1), */ 0x15, 0xFF, /* Logical Minimum (-1), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x09, 0x30, /* Usage (X), */ 0x09, 0x31, /* Usage (Y), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x2E, /* Report Count (46), */ 0x81, 0x01, /* Input (Constant), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_v2_frame_dial_size = sizeof(uclogic_rdesc_v2_frame_dial_arr); const __u8 uclogic_ugee_v2_probe_arr[] = { 0x02, 0xb0, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; const size_t uclogic_ugee_v2_probe_size = sizeof(uclogic_ugee_v2_probe_arr); const int uclogic_ugee_v2_probe_endpoint = 0x03; /* Fixed report descriptor template for UGEE v2 pen reports */ const __u8 uclogic_rdesc_ugee_v2_pen_template_arr[] = { 0x05, 0x0d, /* Usage Page (Digitizers), */ 0x09, 0x01, /* Usage (Digitizer), */ 0xa1, 0x01, /* Collection (Application), */ 0x85, 0x02, /* Report ID (2), */ 0x09, 0x20, /* Usage (Stylus), */ 0xa1, 0x00, /* Collection (Physical), */ 0x09, 0x42, /* Usage (Tip Switch), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x09, 0x46, /* Usage (Tablet Pick), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x03, /* Report Count (3), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x09, 0x32, /* Usage (In Range), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x75, 0x10, /* Report Size (16), */ 0x95, 0x01, /* Report Count (1), */ 0x35, 0x00, /* Physical Minimum (0), */ 0xa4, /* Push, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x30, /* Usage (X), */ 0x65, 0x13, /* Unit (Inch), */ 0x55, 0x0d, /* Unit Exponent (-3), */ 0x27, UCLOGIC_RDESC_PEN_PH(X_LM), /* Logical Maximum (PLACEHOLDER), */ 0x47, UCLOGIC_RDESC_PEN_PH(X_PM), /* Physical Maximum (PLACEHOLDER), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x31, /* Usage (Y), */ 0x27, UCLOGIC_RDESC_PEN_PH(Y_LM), /* Logical Maximum (PLACEHOLDER), */ 0x47, UCLOGIC_RDESC_PEN_PH(Y_PM), /* Physical Maximum (PLACEHOLDER), */ 0x81, 0x02, /* Input (Variable), */ 0xb4, /* Pop, */ 0x09, 0x30, /* Usage (Tip Pressure), */ 0x45, 0x00, /* Physical Maximum (0), */ 0x27, UCLOGIC_RDESC_PEN_PH(PRESSURE_LM), /* Logical Maximum (PLACEHOLDER), */ 0x75, 0x0D, /* Report Size (13), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x01, /* Input (Constant), */ 0x09, 0x3d, /* Usage (X Tilt), */ 0x35, 0xC3, /* Physical Minimum (-61), */ 0x45, 0x3C, /* Physical Maximum (60), */ 0x15, 0xC3, /* Logical Minimum (-61), */ 0x25, 0x3C, /* Logical Maximum (60), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x3e, /* Usage (Y Tilt), */ 0x35, 0xC3, /* Physical Minimum (-61), */ 0x45, 0x3C, /* Physical Maximum (60), */ 0x15, 0xC3, /* Logical Minimum (-61), */ 0x25, 0x3C, /* Logical Maximum (60), */ 0x81, 0x02, /* Input (Variable), */ 0xc0, /* End Collection, */ 0xc0, /* End Collection */ }; const size_t uclogic_rdesc_ugee_v2_pen_template_size = sizeof(uclogic_rdesc_ugee_v2_pen_template_arr); /* Fixed report descriptor template for UGEE v2 frame reports (buttons only) */ const __u8 uclogic_rdesc_ugee_v2_frame_btn_template_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x07, /* Usage (Keypad), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, UCLOGIC_RDESC_V1_FRAME_ID, /* Report ID, */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x39, /* Usage (Tablet Function Keys), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x08, /* Report Count (8), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x09, /* Usage Page (Button), */ 0x19, 0x01, /* Usage Minimum (01h), */ UCLOGIC_RDESC_FRAME_PH_BTN, /* Usage Maximum (PLACEHOLDER), */ 0x95, 0x0A, /* Report Count (10), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x46, /* Report Count (70), */ 0x81, 0x01, /* Input (Constant), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_ugee_v2_frame_btn_template_size = sizeof(uclogic_rdesc_ugee_v2_frame_btn_template_arr); /* Fixed report descriptor template for UGEE v2 frame reports (dial) */ const __u8 uclogic_rdesc_ugee_v2_frame_dial_template_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x07, /* Usage (Keypad), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, UCLOGIC_RDESC_V1_FRAME_ID, /* Report ID, */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x39, /* Usage (Tablet Function Keys), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x08, /* Report Count (8), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x09, /* Usage Page (Button), */ 0x19, 0x01, /* Usage Minimum (01h), */ UCLOGIC_RDESC_FRAME_PH_BTN, /* Usage Maximum (PLACEHOLDER), */ 0x95, 0x0A, /* Report Count (10), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x06, /* Report Count (6), */ 0x81, 0x01, /* Input (Constant), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x38, /* Usage (Wheel), */ 0x95, 0x01, /* Report Count (1), */ 0x15, 0xFF, /* Logical Minimum (-1), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x01, /* Input (Constant), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_ugee_v2_frame_dial_template_size = sizeof(uclogic_rdesc_ugee_v2_frame_dial_template_arr); /* Fixed report descriptor template for UGEE v2 frame reports (mouse) */ const __u8 uclogic_rdesc_ugee_v2_frame_mouse_template_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x02, /* Usage (Mouse), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x01, /* Report ID (1), */ 0x05, 0x01, /* Usage Page (Pointer), */ 0xA0, /* Collection (Physical), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x02, /* Report Count (2), */ 0x05, 0x09, /* Usage Page (Button), */ 0x19, 0x01, /* Usage Minimum (01h), */ 0x29, 0x02, /* Usage Maximum (02h), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x06, /* Report Count (6), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x01, /* Usage Page (Generic Desktop), */ 0x09, 0x30, /* Usage (X), */ 0x09, 0x31, /* Usage (Y), */ 0x75, 0x10, /* Report Size (16), */ 0x95, 0x02, /* Report Count (2), */ 0x16, 0x00, 0x80, /* Logical Minimum (-32768), */ 0x26, 0xFF, 0x7F, /* Logical Maximum (32767), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x01, /* Input (Constant), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_ugee_v2_frame_mouse_template_size = sizeof(uclogic_rdesc_ugee_v2_frame_mouse_template_arr); /* Fixed report descriptor template for UGEE v2 battery reports */ const __u8 uclogic_rdesc_ugee_v2_battery_template_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x07, /* Usage (Keypad), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, UCLOGIC_RDESC_UGEE_V2_BATTERY_ID, /* Report ID, */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x84, /* Usage Page (Power Device), */ 0x05, 0x85, /* Usage Page (Battery System), */ 0x09, 0x65, /* Usage Page (AbsoluteStateOfCharge), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x01, /* Report Count (1), */ 0x15, 0x00, /* Logical Minimum (0), */ 0x26, 0xff, 0x00, /* Logical Maximum (255), */ 0x81, 0x02, /* Input (Variable), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x01, /* Report Count (1), */ 0x15, 0x00, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x09, 0x44, /* Usage Page (Charging), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x07, /* Report Count (7), */ 0x81, 0x01, /* Input (Constant), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x07, /* Report Count (7), */ 0x81, 0x01, /* Input (Constant), */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_ugee_v2_battery_template_size = sizeof(uclogic_rdesc_ugee_v2_battery_template_arr); /* Fixed report descriptor for Ugee EX07 frame */ const __u8 uclogic_rdesc_ugee_ex07_frame_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x07, /* Usage (Keypad), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x06, /* Report ID (6), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x39, /* Usage (Tablet Function Keys), */ 0xA0, /* Collection (Physical), */ 0x05, 0x09, /* Usage Page (Button), */ 0x75, 0x01, /* Report Size (1), */ 0x19, 0x03, /* Usage Minimum (03h), */ 0x29, 0x06, /* Usage Maximum (06h), */ 0x95, 0x04, /* Report Count (4), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x1A, /* Report Count (26), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x19, 0x01, /* Usage Minimum (01h), */ 0x29, 0x02, /* Usage Maximum (02h), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x02, /* Input (Variable), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_ugee_ex07_frame_size = sizeof(uclogic_rdesc_ugee_ex07_frame_arr); /* Fixed report descriptor for Ugee G5 frame controls */ const __u8 uclogic_rdesc_ugee_g5_frame_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x07, /* Usage (Keypad), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x06, /* Report ID (6), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x39, /* Usage (Tablet Function Keys), */ 0xA0, /* Collection (Physical), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x05, 0x09, /* Usage Page (Button), */ 0x19, 0x01, /* Usage Minimum (01h), */ 0x29, 0x05, /* Usage Maximum (05h), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x05, /* Report Count (5), */ 0x81, 0x02, /* Input (Variable), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x0A, 0xFF, 0xFF, /* Usage (FFFFh), */ 0x26, 0xFF, 0x00, /* Logical Maximum (255), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x30, /* Usage (X), */ 0x09, 0x31, /* Usage (Y), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x02, /* Input (Variable), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x0B, /* Report Count (11), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x38, /* Usage (Wheel), */ 0x15, 0xFF, /* Logical Minimum (-1), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x02, /* Report Size (2), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_ugee_g5_frame_size = sizeof(uclogic_rdesc_ugee_g5_frame_arr); /* Fixed report descriptor for XP-Pen Deco 01 frame controls */ const __u8 uclogic_rdesc_xppen_deco01_frame_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x07, /* Usage (Keypad), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, 0x06, /* Report ID (6), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x39, /* Usage (Tablet Function Keys), */ 0xA0, /* Collection (Physical), */ 0x05, 0x09, /* Usage Page (Button), */ 0x19, 0x01, /* Usage Minimum (01h), */ 0x29, 0x08, /* Usage Maximum (08h), */ 0x95, 0x08, /* Report Count (8), */ 0x81, 0x02, /* Input (Variable), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x30, /* Usage (X), */ 0x09, 0x31, /* Usage (Y), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x15, /* Report Count (21), */ 0x81, 0x01, /* Input (Constant), */ 0xC0, /* End Collection, */ 0xC0 /* End Collection */ }; const size_t uclogic_rdesc_xppen_deco01_frame_size = sizeof(uclogic_rdesc_xppen_deco01_frame_arr); /* Fixed report descriptor for XP-Pen Arist 22R Pro frame */ const __u8 uclogic_rdesc_xppen_artist_22r_pro_frame_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x07, /* Usage (Keypad), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, UCLOGIC_RDESC_V1_FRAME_ID, /* Report ID (Virtual report), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x39, /* Usage (Tablet Function Keys), */ 0xA0, /* Collection (Physical), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x08, /* Report Count (8), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x09, /* Usage Page (Button), */ 0x19, 0x01, /* Usage Minimum (01h), */ 0x29, 0x14, /* Usage Maximum (14h), */ 0x95, 0x14, /* Report Count (20), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x14, /* Report Count (20), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x38, /* Usage (Wheel), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x01, /* Report Count (1), */ 0x15, 0xFF, /* Logical Minimum (-1), */ 0x25, 0x08, /* Logical Maximum (8), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x05, 0x0C, /* Usage Page (Consumer Devices), */ 0x0A, 0x38, 0x02, /* Usage (AC PAN), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x26, 0xFF, 0x00, /* Logical Maximum (255), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0xC0, /* End Collection */ 0xC0, /* End Collection */ }; const size_t uclogic_rdesc_xppen_artist_22r_pro_frame_size = sizeof(uclogic_rdesc_xppen_artist_22r_pro_frame_arr); /* Fixed report descriptor template for XP-PEN 24 Pro reports * Mostly identical to uclogic_rdesc_ugee_v2_pen_template_arr except that the X coordinate has to be * 32-bits instead of 16-bits. */ const __u8 uclogic_rdesc_xppen_artist_24_pro_pen_template_arr[] = { 0x05, 0x0d, /* Usage Page (Digitizers), */ 0x09, 0x01, /* Usage (Digitizer), */ 0xa1, 0x01, /* Collection (Application), */ 0x85, 0x02, /* Report ID (2), */ 0x09, 0x20, /* Usage (Stylus), */ 0xa1, 0x00, /* Collection (Physical), */ 0x09, 0x42, /* Usage (Tip Switch), */ 0x09, 0x44, /* Usage (Barrel Switch), */ 0x09, 0x46, /* Usage (Tablet Pick), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x03, /* Report Count (3), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x09, 0x32, /* Usage (In Range), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x02, /* Report Count (2), */ 0x81, 0x03, /* Input (Constant, Variable), */ 0x75, 0x10, /* Report Size (16), */ 0x95, 0x01, /* Report Count (1), */ 0x35, 0x00, /* Physical Minimum (0), */ 0xa4, /* Push, */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x30, /* Usage (X), */ 0x65, 0x13, /* Unit (Inch), */ 0x55, 0x0d, /* Unit Exponent (-3), */ 0x27, UCLOGIC_RDESC_PEN_PH(X_LM), /* Logical Maximum (PLACEHOLDER), */ 0x47, UCLOGIC_RDESC_PEN_PH(X_PM), /* Physical Maximum (PLACEHOLDER), */ 0x75, 0x20, /* Report Size (32), */ 0x81, 0x02, /* Input (Variable), */ 0x75, 0x10, /* Report Size (16), */ 0x09, 0x31, /* Usage (Y), */ 0x27, UCLOGIC_RDESC_PEN_PH(Y_LM), /* Logical Maximum (PLACEHOLDER), */ 0x47, UCLOGIC_RDESC_PEN_PH(Y_PM), /* Physical Maximum (PLACEHOLDER), */ 0x81, 0x02, /* Input (Variable), */ 0xb4, /* Pop, */ 0x09, 0x30, /* Usage (Tip Pressure), */ 0x45, 0x00, /* Physical Maximum (0), */ 0x27, UCLOGIC_RDESC_PEN_PH(PRESSURE_LM), /* Logical Maximum (PLACEHOLDER), */ 0x75, 0x0D, /* Report Size (13), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x03, /* Report Count (3), */ 0x81, 0x01, /* Input (Constant), */ 0x09, 0x3d, /* Usage (X Tilt), */ 0x35, 0xC3, /* Physical Minimum (-61), */ 0x45, 0x3C, /* Physical Maximum (60), */ 0x15, 0xC3, /* Logical Minimum (-61), */ 0x25, 0x3C, /* Logical Maximum (60), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x09, 0x3e, /* Usage (Y Tilt), */ 0x35, 0xC3, /* Physical Minimum (-61), */ 0x45, 0x3C, /* Physical Maximum (60), */ 0x15, 0xC3, /* Logical Minimum (-61), */ 0x25, 0x3C, /* Logical Maximum (60), */ 0x81, 0x02, /* Input (Variable), */ 0xc0, /* End Collection, */ 0xc0, /* End Collection */ }; const size_t uclogic_rdesc_xppen_artist_24_pro_pen_template_size = sizeof(uclogic_rdesc_xppen_artist_24_pro_pen_template_arr); /* Fixed report descriptor for XP-Pen Arist 24 Pro frame */ const __u8 uclogic_rdesc_xppen_artist_24_pro_frame_arr[] = { 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x07, /* Usage (Keypad), */ 0xA1, 0x01, /* Collection (Application), */ 0x85, UCLOGIC_RDESC_V1_FRAME_ID, /* Report ID (Virtual report), */ 0x05, 0x0D, /* Usage Page (Digitizer), */ 0x09, 0x39, /* Usage (Tablet Function Keys), */ 0xA0, /* Collection (Physical), */ 0x14, /* Logical Minimum (0), */ 0x25, 0x01, /* Logical Maximum (1), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 0x08, /* Report Count (8), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x09, /* Usage Page (Button), */ 0x19, 0x01, /* Usage Minimum (01h), */ 0x29, 0x14, /* Usage Maximum (14h), */ 0x95, 0x14, /* Report Count (20), */ 0x81, 0x02, /* Input (Variable), */ 0x95, 0x14, /* Report Count (20), */ 0x81, 0x01, /* Input (Constant), */ 0x05, 0x01, /* Usage Page (Desktop), */ 0x09, 0x38, /* Usage (Wheel), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x01, /* Report Count (1), */ 0x15, 0xFF, /* Logical Minimum (-1), */ 0x25, 0x08, /* Logical Maximum (8), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x05, 0x0C, /* Usage Page (Consumer Devices), */ 0x0A, 0x38, 0x02, /* Usage (AC PAN), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x06, /* Input (Variable, Relative), */ 0x26, 0xFF, 0x00, /* Logical Maximum (255), */ 0x75, 0x08, /* Report Size (8), */ 0x95, 0x01, /* Report Count (1), */ 0x81, 0x02, /* Input (Variable), */ 0x75, 0x01, /* Report Size (1), */ 0x95, 16, /* Report Count (16), */ 0x81, 0x01, /* Input (Constant), */ 0xC0, /* End Collection */ 0xC0, /* End Collection */ }; const size_t uclogic_rdesc_xppen_artist_24_pro_frame_size = sizeof(uclogic_rdesc_xppen_artist_24_pro_frame_arr); /** * uclogic_rdesc_template_apply() - apply report descriptor parameters to a * report descriptor template, creating a report descriptor. Copies the * template over to the new report descriptor and replaces every occurrence of * the template placeholders, followed by an index byte, with the value from the * parameter list at that index. * * @template_ptr: Pointer to the template buffer. * @template_size: Size of the template buffer. * @param_list: List of template parameters. * @param_num: Number of parameters in the list. * * Returns: * Kmalloc-allocated pointer to the created report descriptor, * or NULL if allocation failed. */ __u8 *uclogic_rdesc_template_apply(const __u8 *template_ptr, size_t template_size, const s32 *param_list, size_t param_num) { static const __u8 btn_head[] = {UCLOGIC_RDESC_FRAME_PH_BTN_HEAD}; static const __u8 pen_head[] = {UCLOGIC_RDESC_PEN_PH_HEAD}; __u8 *rdesc_ptr; __u8 *p; s32 v; rdesc_ptr = kmemdup(template_ptr, template_size, GFP_KERNEL); if (rdesc_ptr == NULL) return NULL; for (p = rdesc_ptr; p + sizeof(btn_head) < rdesc_ptr + template_size;) { if (p + sizeof(pen_head) < rdesc_ptr + template_size && memcmp(p, pen_head, sizeof(pen_head)) == 0 && p[sizeof(pen_head)] < param_num) { v = param_list[p[sizeof(pen_head)]]; put_unaligned((__force u32)cpu_to_le32(v), (s32 *)p); p += sizeof(pen_head) + 1; } else if (memcmp(p, btn_head, sizeof(btn_head)) == 0 && p[sizeof(btn_head)] < param_num) { v = param_list[p[sizeof(btn_head)]]; put_unaligned((__u8)0x2A, p); /* Usage Maximum */ put_unaligned((__force u16)cpu_to_le16(v), (s16 *)(p + 1)); p += sizeof(btn_head) + 1; } else { p++; } } return rdesc_ptr; } EXPORT_SYMBOL_IF_KUNIT(uclogic_rdesc_template_apply); |
| 53 53 3 51 51 50 35 35 35 8 8 27 26 92 53 8 2 8 2 1 6 6 16 2 16 16 2 1 14 14 19 19 4 4 4 4 1 3 1 1 1 3 1 3 3 3 2 2 2 2 1 2 1 23 10 8 37 50 92 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Squashfs - a compressed read only filesystem for Linux * * Copyright (c) 2002, 2003, 2004, 2005, 2006, 2007, 2008 * Phillip Lougher <phillip@squashfs.org.uk> * * inode.c */ /* * This file implements code to create and read inodes from disk. * * Inodes in Squashfs are identified by a 48-bit inode which encodes the * location of the compressed metadata block containing the inode, and the byte * offset into that block where the inode is placed (<block, offset>). * * To maximise compression there are different inodes for each file type * (regular file, directory, device, etc.), the inode contents and length * varying with the type. * * To further maximise compression, two types of regular file inode and * directory inode are defined: inodes optimised for frequently occurring * regular files and directories, and extended types where extra * information has to be stored. */ #include <linux/fs.h> #include <linux/vfs.h> #include <linux/xattr.h> #include <linux/pagemap.h> #include "squashfs_fs.h" #include "squashfs_fs_sb.h" #include "squashfs_fs_i.h" #include "squashfs.h" #include "xattr.h" /* * Initialise VFS inode with the base inode information common to all * Squashfs inode types. Sqsh_ino contains the unswapped base inode * off disk. */ static int squashfs_new_inode(struct super_block *sb, struct inode *inode, struct squashfs_base_inode *sqsh_ino) { uid_t i_uid; gid_t i_gid; int err; inode->i_ino = le32_to_cpu(sqsh_ino->inode_number); if (inode->i_ino == 0) return -EINVAL; err = squashfs_get_id(sb, le16_to_cpu(sqsh_ino->uid), &i_uid); if (err) return err; err = squashfs_get_id(sb, le16_to_cpu(sqsh_ino->guid), &i_gid); if (err) return err; i_uid_write(inode, i_uid); i_gid_write(inode, i_gid); inode_set_mtime(inode, le32_to_cpu(sqsh_ino->mtime), 0); inode_set_atime(inode, inode_get_mtime_sec(inode), 0); inode_set_ctime(inode, inode_get_mtime_sec(inode), 0); inode->i_mode = le16_to_cpu(sqsh_ino->mode); inode->i_size = 0; /* File type must not be set at this moment, for it will later be set by the caller. */ if (inode->i_mode & S_IFMT) err = -EIO; return err; } struct inode *squashfs_iget(struct super_block *sb, long long ino, unsigned int ino_number) { struct inode *inode = iget_locked(sb, ino_number); int err; TRACE("Entered squashfs_iget\n"); if (!inode) return ERR_PTR(-ENOMEM); if (!(inode_state_read_once(inode) & I_NEW)) return inode; err = squashfs_read_inode(inode, ino); if (err) { iget_failed(inode); return ERR_PTR(err); } unlock_new_inode(inode); return inode; } /* * Initialise VFS inode by reading inode from inode table (compressed * metadata). The format and amount of data read depends on type. */ int squashfs_read_inode(struct inode *inode, long long ino) { struct super_block *sb = inode->i_sb; struct squashfs_sb_info *msblk = sb->s_fs_info; u64 block = SQUASHFS_INODE_BLK(ino) + msblk->inode_table; int err, type, offset = SQUASHFS_INODE_OFFSET(ino); union squashfs_inode squashfs_ino; struct squashfs_base_inode *sqshb_ino = &squashfs_ino.base; int xattr_id = SQUASHFS_INVALID_XATTR; TRACE("Entered squashfs_read_inode\n"); /* * Read inode base common to all inode types. */ err = squashfs_read_metadata(sb, sqshb_ino, &block, &offset, sizeof(*sqshb_ino)); if (err < 0) goto failed_read; err = squashfs_new_inode(sb, inode, sqshb_ino); if (err) goto failed_read; block = SQUASHFS_INODE_BLK(ino) + msblk->inode_table; offset = SQUASHFS_INODE_OFFSET(ino); type = le16_to_cpu(sqshb_ino->inode_type); switch (type) { case SQUASHFS_REG_TYPE: { unsigned int frag_offset, frag; int frag_size; u64 frag_blk; struct squashfs_reg_inode *sqsh_ino = &squashfs_ino.reg; err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, sizeof(*sqsh_ino)); if (err < 0) goto failed_read; inode->i_size = le32_to_cpu(sqsh_ino->file_size); frag = le32_to_cpu(sqsh_ino->fragment); if (frag != SQUASHFS_INVALID_FRAG) { /* * the file cannot have a fragment (tailend) and have a * file size a multiple of the block size */ if ((inode->i_size & (msblk->block_size - 1)) == 0) { err = -EINVAL; goto failed_read; } frag_offset = le32_to_cpu(sqsh_ino->offset); frag_size = squashfs_frag_lookup(sb, frag, &frag_blk); if (frag_size < 0) { err = frag_size; goto failed_read; } } else { frag_blk = SQUASHFS_INVALID_BLK; frag_size = 0; frag_offset = 0; } set_nlink(inode, 1); inode->i_fop = &squashfs_file_operations; inode->i_mode |= S_IFREG; inode->i_blocks = ((inode->i_size - 1) >> 9) + 1; squashfs_i(inode)->fragment_block = frag_blk; squashfs_i(inode)->fragment_size = frag_size; squashfs_i(inode)->fragment_offset = frag_offset; squashfs_i(inode)->start = le32_to_cpu(sqsh_ino->start_block); squashfs_i(inode)->block_list_start = block; squashfs_i(inode)->offset = offset; squashfs_i(inode)->parent = 0; inode->i_data.a_ops = &squashfs_aops; TRACE("File inode %x:%x, start_block %llx, block_list_start " "%llx, offset %x\n", SQUASHFS_INODE_BLK(ino), offset, squashfs_i(inode)->start, block, offset); break; } case SQUASHFS_LREG_TYPE: { unsigned int frag_offset, frag; int frag_size; u64 frag_blk; struct squashfs_lreg_inode *sqsh_ino = &squashfs_ino.lreg; err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, sizeof(*sqsh_ino)); if (err < 0) goto failed_read; inode->i_size = le64_to_cpu(sqsh_ino->file_size); if (inode->i_size < 0) { err = -EINVAL; goto failed_read; } frag = le32_to_cpu(sqsh_ino->fragment); if (frag != SQUASHFS_INVALID_FRAG) { /* * the file cannot have a fragment (tailend) and have a * file size a multiple of the block size */ if ((inode->i_size & (msblk->block_size - 1)) == 0) { err = -EINVAL; goto failed_read; } frag_offset = le32_to_cpu(sqsh_ino->offset); frag_size = squashfs_frag_lookup(sb, frag, &frag_blk); if (frag_size < 0) { err = frag_size; goto failed_read; } } else { frag_blk = SQUASHFS_INVALID_BLK; frag_size = 0; frag_offset = 0; } xattr_id = le32_to_cpu(sqsh_ino->xattr); set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); inode->i_op = &squashfs_inode_ops; inode->i_fop = &squashfs_file_operations; inode->i_mode |= S_IFREG; inode->i_blocks = (inode->i_size - le64_to_cpu(sqsh_ino->sparse) + 511) >> 9; squashfs_i(inode)->fragment_block = frag_blk; squashfs_i(inode)->fragment_size = frag_size; squashfs_i(inode)->fragment_offset = frag_offset; squashfs_i(inode)->start = le64_to_cpu(sqsh_ino->start_block); squashfs_i(inode)->block_list_start = block; squashfs_i(inode)->offset = offset; squashfs_i(inode)->parent = 0; inode->i_data.a_ops = &squashfs_aops; TRACE("File inode %x:%x, start_block %llx, block_list_start " "%llx, offset %x\n", SQUASHFS_INODE_BLK(ino), offset, squashfs_i(inode)->start, block, offset); break; } case SQUASHFS_DIR_TYPE: { struct squashfs_dir_inode *sqsh_ino = &squashfs_ino.dir; err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, sizeof(*sqsh_ino)); if (err < 0) goto failed_read; set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); inode->i_size = le16_to_cpu(sqsh_ino->file_size); inode->i_op = &squashfs_dir_inode_ops; inode->i_fop = &squashfs_dir_ops; inode->i_mode |= S_IFDIR; squashfs_i(inode)->start = le32_to_cpu(sqsh_ino->start_block); squashfs_i(inode)->offset = le16_to_cpu(sqsh_ino->offset); squashfs_i(inode)->dir_idx_cnt = 0; squashfs_i(inode)->parent = le32_to_cpu(sqsh_ino->parent_inode); TRACE("Directory inode %x:%x, start_block %llx, offset %x\n", SQUASHFS_INODE_BLK(ino), offset, squashfs_i(inode)->start, le16_to_cpu(sqsh_ino->offset)); break; } case SQUASHFS_LDIR_TYPE: { struct squashfs_ldir_inode *sqsh_ino = &squashfs_ino.ldir; err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, sizeof(*sqsh_ino)); if (err < 0) goto failed_read; xattr_id = le32_to_cpu(sqsh_ino->xattr); set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); inode->i_size = le32_to_cpu(sqsh_ino->file_size); inode->i_op = &squashfs_dir_inode_ops; inode->i_fop = &squashfs_dir_ops; inode->i_mode |= S_IFDIR; squashfs_i(inode)->start = le32_to_cpu(sqsh_ino->start_block); squashfs_i(inode)->offset = le16_to_cpu(sqsh_ino->offset); squashfs_i(inode)->dir_idx_start = block; squashfs_i(inode)->dir_idx_offset = offset; squashfs_i(inode)->dir_idx_cnt = le16_to_cpu(sqsh_ino->i_count); squashfs_i(inode)->parent = le32_to_cpu(sqsh_ino->parent_inode); TRACE("Long directory inode %x:%x, start_block %llx, offset " "%x\n", SQUASHFS_INODE_BLK(ino), offset, squashfs_i(inode)->start, le16_to_cpu(sqsh_ino->offset)); break; } case SQUASHFS_SYMLINK_TYPE: case SQUASHFS_LSYMLINK_TYPE: { struct squashfs_symlink_inode *sqsh_ino = &squashfs_ino.symlink; err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, sizeof(*sqsh_ino)); if (err < 0) goto failed_read; inode->i_size = le32_to_cpu(sqsh_ino->symlink_size); if (inode->i_size > PAGE_SIZE) { ERROR("Corrupted symlink\n"); return -EINVAL; } set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); inode->i_op = &squashfs_symlink_inode_ops; inode_nohighmem(inode); inode->i_data.a_ops = &squashfs_symlink_aops; inode->i_mode |= S_IFLNK; squashfs_i(inode)->start = block; squashfs_i(inode)->offset = offset; squashfs_i(inode)->parent = 0; if (type == SQUASHFS_LSYMLINK_TYPE) { __le32 xattr; err = squashfs_read_metadata(sb, NULL, &block, &offset, inode->i_size); if (err < 0) goto failed_read; err = squashfs_read_metadata(sb, &xattr, &block, &offset, sizeof(xattr)); if (err < 0) goto failed_read; xattr_id = le32_to_cpu(xattr); } TRACE("Symbolic link inode %x:%x, start_block %llx, offset " "%x\n", SQUASHFS_INODE_BLK(ino), offset, block, offset); break; } case SQUASHFS_BLKDEV_TYPE: case SQUASHFS_CHRDEV_TYPE: { struct squashfs_dev_inode *sqsh_ino = &squashfs_ino.dev; unsigned int rdev; err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, sizeof(*sqsh_ino)); if (err < 0) goto failed_read; if (type == SQUASHFS_CHRDEV_TYPE) inode->i_mode |= S_IFCHR; else inode->i_mode |= S_IFBLK; set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); rdev = le32_to_cpu(sqsh_ino->rdev); init_special_inode(inode, inode->i_mode, new_decode_dev(rdev)); squashfs_i(inode)->parent = 0; TRACE("Device inode %x:%x, rdev %x\n", SQUASHFS_INODE_BLK(ino), offset, rdev); break; } case SQUASHFS_LBLKDEV_TYPE: case SQUASHFS_LCHRDEV_TYPE: { struct squashfs_ldev_inode *sqsh_ino = &squashfs_ino.ldev; unsigned int rdev; err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, sizeof(*sqsh_ino)); if (err < 0) goto failed_read; if (type == SQUASHFS_LCHRDEV_TYPE) inode->i_mode |= S_IFCHR; else inode->i_mode |= S_IFBLK; xattr_id = le32_to_cpu(sqsh_ino->xattr); inode->i_op = &squashfs_inode_ops; set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); rdev = le32_to_cpu(sqsh_ino->rdev); init_special_inode(inode, inode->i_mode, new_decode_dev(rdev)); squashfs_i(inode)->parent = 0; TRACE("Device inode %x:%x, rdev %x\n", SQUASHFS_INODE_BLK(ino), offset, rdev); break; } case SQUASHFS_FIFO_TYPE: case SQUASHFS_SOCKET_TYPE: { struct squashfs_ipc_inode *sqsh_ino = &squashfs_ino.ipc; err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, sizeof(*sqsh_ino)); if (err < 0) goto failed_read; if (type == SQUASHFS_FIFO_TYPE) inode->i_mode |= S_IFIFO; else inode->i_mode |= S_IFSOCK; set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); init_special_inode(inode, inode->i_mode, 0); squashfs_i(inode)->parent = 0; break; } case SQUASHFS_LFIFO_TYPE: case SQUASHFS_LSOCKET_TYPE: { struct squashfs_lipc_inode *sqsh_ino = &squashfs_ino.lipc; err = squashfs_read_metadata(sb, sqsh_ino, &block, &offset, sizeof(*sqsh_ino)); if (err < 0) goto failed_read; if (type == SQUASHFS_LFIFO_TYPE) inode->i_mode |= S_IFIFO; else inode->i_mode |= S_IFSOCK; xattr_id = le32_to_cpu(sqsh_ino->xattr); inode->i_op = &squashfs_inode_ops; set_nlink(inode, le32_to_cpu(sqsh_ino->nlink)); init_special_inode(inode, inode->i_mode, 0); squashfs_i(inode)->parent = 0; break; } default: ERROR("Unknown inode type %d in squashfs_iget!\n", type); return -EINVAL; } if (xattr_id != SQUASHFS_INVALID_XATTR && msblk->xattr_id_table) { err = squashfs_xattr_lookup(sb, xattr_id, &squashfs_i(inode)->xattr_count, &squashfs_i(inode)->xattr_size, &squashfs_i(inode)->xattr); if (err < 0) goto failed_read; inode->i_blocks += ((squashfs_i(inode)->xattr_size - 1) >> 9) + 1; } else squashfs_i(inode)->xattr_count = 0; return 0; failed_read: ERROR("Unable to read inode 0x%llx\n", ino); return err; } const struct inode_operations squashfs_inode_ops = { .listxattr = squashfs_listxattr }; |
| 17 4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 | // SPDX-License-Identifier: GPL-2.0-only /* * vivid-cec.c - A Virtual Video Test Driver, cec emulation * * Copyright 2016 Cisco Systems, Inc. and/or its affiliates. All rights reserved. */ #include <linux/delay.h> #include <media/cec.h> #include "vivid-core.h" #include "vivid-cec.h" #define CEC_START_BIT_US 4500 #define CEC_DATA_BIT_US 2400 #define CEC_MARGIN_US 350 struct xfer_on_bus { struct cec_adapter *adap; u8 status; }; static bool find_dest_adap(struct vivid_dev *dev, struct cec_adapter *adap, u8 dest) { unsigned int i, j; if (dest >= 0xf) return false; if (adap != dev->cec_rx_adap && dev->cec_rx_adap && dev->cec_rx_adap->is_configured && cec_has_log_addr(dev->cec_rx_adap, dest)) return true; for (i = 0, j = 0; i < dev->num_inputs; i++) { unsigned int menu_idx = dev->input_is_connected_to_output[i]; if (dev->input_type[i] != HDMI) continue; j++; if (menu_idx < FIXED_MENU_ITEMS) continue; struct vivid_dev *dev_tx = vivid_ctrl_hdmi_to_output_instance[menu_idx]; unsigned int output = vivid_ctrl_hdmi_to_output_index[menu_idx]; if (!dev_tx) continue; unsigned int hdmi_output = dev_tx->output_to_iface_index[output]; if (adap == dev_tx->cec_tx_adap[hdmi_output]) continue; if (!dev_tx->cec_tx_adap[hdmi_output]->is_configured) continue; if (cec_has_log_addr(dev_tx->cec_tx_adap[hdmi_output], dest)) return true; } return false; } static bool xfer_ready(struct vivid_dev *dev) { unsigned int i; bool ready = false; spin_lock(&dev->cec_xfers_slock); for (i = 0; i < ARRAY_SIZE(dev->xfers); i++) { if (dev->xfers[i].sft && dev->xfers[i].sft <= dev->cec_sft) { ready = true; break; } } spin_unlock(&dev->cec_xfers_slock); return ready; } /* * If an adapter tries to send successive messages, it must wait for the * longest signal-free time between its transmissions. But, if another * adapter sends a message in the interim, then the wait can be reduced * because the messages are no longer successive. Make these adjustments * if necessary. Should be called holding cec_xfers_slock. */ static void adjust_sfts(struct vivid_dev *dev) { unsigned int i; u8 initiator; for (i = 0; i < ARRAY_SIZE(dev->xfers); i++) { if (dev->xfers[i].sft <= CEC_SIGNAL_FREE_TIME_RETRY) continue; initiator = dev->xfers[i].msg[0] >> 4; if (initiator == dev->last_initiator) dev->xfers[i].sft = CEC_SIGNAL_FREE_TIME_NEXT_XFER; else dev->xfers[i].sft = CEC_SIGNAL_FREE_TIME_NEW_INITIATOR; } } /* * The main emulation of the bus on which CEC adapters attempt to send * messages to each other. The bus keeps track of how long it has been * signal-free and accepts a pending transmission only if the state of * the bus matches the transmission's signal-free requirements. It calls * cec_transmit_attempt_done() for all transmits that enter the bus and * cec_received_msg() for successful transmits. */ int vivid_cec_bus_thread(void *_dev) { u32 last_sft; unsigned int i, j; unsigned int dest; ktime_t start, end; s64 delta_us, retry_us; struct vivid_dev *dev = _dev; dev->cec_sft = CEC_SIGNAL_FREE_TIME_NEXT_XFER; for (;;) { bool first = true; int wait_xfer_us = 0; bool valid_dest = false; int wait_arb_lost_us = 0; unsigned int first_idx = 0; unsigned int first_status = 0; struct cec_msg first_msg = {}; struct xfer_on_bus xfers_on_bus[MAX_OUTPUTS] = {}; wait_event_interruptible(dev->kthread_waitq_cec, xfer_ready(dev) || kthread_should_stop()); if (kthread_should_stop()) break; last_sft = dev->cec_sft; dev->cec_sft = 0; /* * Move the messages that are ready onto the bus. The adapter with * the most leading zeros will win control of the bus and any other * adapters will lose arbitration. */ spin_lock(&dev->cec_xfers_slock); for (i = 0; i < ARRAY_SIZE(dev->xfers); i++) { if (!dev->xfers[i].sft || dev->xfers[i].sft > last_sft) continue; if (first) { first = false; first_idx = i; xfers_on_bus[first_idx].adap = dev->xfers[i].adap; memcpy(first_msg.msg, dev->xfers[i].msg, dev->xfers[i].len); first_msg.len = dev->xfers[i].len; } else { xfers_on_bus[i].adap = dev->xfers[i].adap; xfers_on_bus[i].status = CEC_TX_STATUS_ARB_LOST; /* * For simplicity wait for all 4 bits of the initiator's * address even though HDMI specification uses bit-level * precision. */ wait_arb_lost_us = 4 * CEC_DATA_BIT_US + CEC_START_BIT_US; } dev->xfers[i].sft = 0; } dev->last_initiator = cec_msg_initiator(&first_msg); adjust_sfts(dev); spin_unlock(&dev->cec_xfers_slock); dest = cec_msg_destination(&first_msg); valid_dest = cec_msg_is_broadcast(&first_msg); if (!valid_dest) valid_dest = find_dest_adap(dev, xfers_on_bus[first_idx].adap, dest); if (valid_dest) { first_status = CEC_TX_STATUS_OK; /* * Message length is in bytes, but each byte is transmitted in * a block of 10 bits. */ wait_xfer_us = first_msg.len * 10 * CEC_DATA_BIT_US; } else { first_status = CEC_TX_STATUS_NACK; /* * A message that is not acknowledged stops transmitting after * the header block of 10 bits. */ wait_xfer_us = 10 * CEC_DATA_BIT_US; } wait_xfer_us += CEC_START_BIT_US; xfers_on_bus[first_idx].status = first_status; /* Sleep as if sending messages on a real hardware bus. */ start = ktime_get(); if (wait_arb_lost_us) { usleep_range(wait_arb_lost_us - CEC_MARGIN_US, wait_arb_lost_us); for (i = 0; i < ARRAY_SIZE(xfers_on_bus); i++) { if (xfers_on_bus[i].status != CEC_TX_STATUS_ARB_LOST) continue; cec_transmit_attempt_done(xfers_on_bus[i].adap, CEC_TX_STATUS_ARB_LOST); } if (kthread_should_stop()) break; } wait_xfer_us -= wait_arb_lost_us; usleep_range(wait_xfer_us - CEC_MARGIN_US, wait_xfer_us); cec_transmit_attempt_done(xfers_on_bus[first_idx].adap, first_status); if (kthread_should_stop()) break; if (first_status == CEC_TX_STATUS_OK) { if (xfers_on_bus[first_idx].adap != dev->cec_rx_adap) cec_received_msg(dev->cec_rx_adap, &first_msg); for (i = 0, j = 0; i < dev->num_inputs; i++) { unsigned int menu_idx = dev->input_is_connected_to_output[i]; if (dev->input_type[i] != HDMI) continue; j++; if (menu_idx < FIXED_MENU_ITEMS) continue; struct vivid_dev *dev_tx = vivid_ctrl_hdmi_to_output_instance[menu_idx]; unsigned int output = vivid_ctrl_hdmi_to_output_index[menu_idx]; if (!dev_tx) continue; unsigned int hdmi_output = dev_tx->output_to_iface_index[output]; if (xfers_on_bus[first_idx].adap != dev_tx->cec_tx_adap[hdmi_output]) cec_received_msg(dev_tx->cec_tx_adap[hdmi_output], &first_msg); } } end = ktime_get(); /* * If the emulated transfer took more or less time than it should * have, then compensate by adjusting the wait time needed for the * bus to be signal-free for 3 bit periods (the retry time). */ delta_us = div_s64(end - start, 1000); delta_us -= wait_xfer_us + wait_arb_lost_us; retry_us = CEC_SIGNAL_FREE_TIME_RETRY * CEC_DATA_BIT_US - delta_us; if (retry_us > CEC_MARGIN_US) usleep_range(retry_us - CEC_MARGIN_US, retry_us); dev->cec_sft = CEC_SIGNAL_FREE_TIME_RETRY; /* * If there are no messages that need to be retried, check if any * adapters that did not just transmit a message are ready to * transmit. If none of these adapters are ready, then increase * the signal-free time so that the bus is available to all * adapters and go back to waiting for a transmission. */ while (dev->cec_sft >= CEC_SIGNAL_FREE_TIME_RETRY && dev->cec_sft < CEC_SIGNAL_FREE_TIME_NEXT_XFER && !xfer_ready(dev) && !kthread_should_stop()) { usleep_range(2 * CEC_DATA_BIT_US - CEC_MARGIN_US, 2 * CEC_DATA_BIT_US); dev->cec_sft += 2; } } return 0; } static int vivid_cec_adap_enable(struct cec_adapter *adap, bool enable) { adap->cec_pin_is_high = true; return 0; } static int vivid_cec_adap_log_addr(struct cec_adapter *adap, u8 log_addr) { return 0; } static int vivid_cec_adap_transmit(struct cec_adapter *adap, u8 attempts, u32 signal_free_time, struct cec_msg *msg) { struct vivid_dev *dev = cec_get_drvdata(adap); struct vivid_dev *dev_rx = dev; u8 idx = cec_msg_initiator(msg); u8 output = 0; if (dev->cec_rx_adap != adap) { int i; for (i = 0; i < dev->num_hdmi_outputs; i++) if (dev->cec_tx_adap[i] == adap) break; if (i == dev->num_hdmi_outputs) return -ENONET; output = dev->hdmi_index_to_output_index[i]; dev_rx = dev->output_to_input_instance[output]; if (!dev_rx) return -ENONET; } spin_lock(&dev_rx->cec_xfers_slock); dev_rx->xfers[idx].adap = adap; memcpy(dev_rx->xfers[idx].msg, msg->msg, CEC_MAX_MSG_SIZE); dev_rx->xfers[idx].len = msg->len; dev_rx->xfers[idx].sft = CEC_SIGNAL_FREE_TIME_RETRY; if (signal_free_time > CEC_SIGNAL_FREE_TIME_RETRY) { if (idx == dev_rx->last_initiator) dev_rx->xfers[idx].sft = CEC_SIGNAL_FREE_TIME_NEXT_XFER; else dev_rx->xfers[idx].sft = CEC_SIGNAL_FREE_TIME_NEW_INITIATOR; } spin_unlock(&dev_rx->cec_xfers_slock); wake_up_interruptible(&dev_rx->kthread_waitq_cec); return 0; } static int vivid_received(struct cec_adapter *adap, struct cec_msg *msg) { struct vivid_dev *dev = cec_get_drvdata(adap); struct cec_msg reply; u8 dest = cec_msg_destination(msg); if (cec_msg_is_broadcast(msg)) dest = adap->log_addrs.log_addr[0]; cec_msg_init(&reply, dest, cec_msg_initiator(msg)); switch (cec_msg_opcode(msg)) { case CEC_MSG_SET_OSD_STRING: { u8 disp_ctl; char osd[14]; if (!cec_is_sink(adap)) break; cec_ops_set_osd_string(msg, &disp_ctl, osd); switch (disp_ctl) { case CEC_OP_DISP_CTL_DEFAULT: strscpy(dev->osd, osd, sizeof(dev->osd)); dev->osd_jiffies = jiffies; break; case CEC_OP_DISP_CTL_UNTIL_CLEARED: strscpy(dev->osd, osd, sizeof(dev->osd)); dev->osd_jiffies = 0; break; case CEC_OP_DISP_CTL_CLEAR: dev->osd[0] = 0; dev->osd_jiffies = 0; break; default: cec_msg_feature_abort(&reply, cec_msg_opcode(msg), CEC_OP_ABORT_INVALID_OP); cec_transmit_msg(adap, &reply, false); break; } return 0; } case CEC_MSG_VENDOR_COMMAND_WITH_ID: { u32 vendor_id; u8 size; const u8 *vendor_cmd; /* * If we receive <Vendor Command With ID> with our vendor ID * and with a payload of size 1, and the payload value is odd, * then we reply with the same message, but with the payload * byte incremented by 1. * * If the size is 1 and the payload value is even, then we * ignore the message. * * The reason we reply to odd instead of even payload values * is that it allows for testing of the corner case where the * reply value is 0 (0xff + 1 % 256). * * For other sizes we Feature Abort. * * This is added for the specific purpose of testing the * CEC_MSG_FL_REPLY_VENDOR_ID flag using vivid. */ cec_ops_vendor_command_with_id(msg, &vendor_id, &size, &vendor_cmd); if (vendor_id != adap->log_addrs.vendor_id) break; if (size == 1) { // Ignore even op values if (!(vendor_cmd[0] & 1)) return 0; reply.len = msg->len; memcpy(reply.msg + 1, msg->msg + 1, msg->len - 1); reply.msg[msg->len - 1]++; } else { cec_msg_feature_abort(&reply, cec_msg_opcode(msg), CEC_OP_ABORT_INVALID_OP); } cec_transmit_msg(adap, &reply, false); return 0; } } return -ENOMSG; } static const struct cec_adap_ops vivid_cec_adap_ops = { .adap_enable = vivid_cec_adap_enable, .adap_log_addr = vivid_cec_adap_log_addr, .adap_transmit = vivid_cec_adap_transmit, .received = vivid_received, }; struct cec_adapter *vivid_cec_alloc_adap(struct vivid_dev *dev, unsigned int idx, bool is_source) { u32 caps = CEC_CAP_DEFAULTS | CEC_CAP_MONITOR_ALL | CEC_CAP_MONITOR_PIN; char name[32]; snprintf(name, sizeof(name), "vivid-%03d-vid-%s%d", dev->inst, is_source ? "out" : "cap", idx); return cec_allocate_adapter(&vivid_cec_adap_ops, dev, name, caps, CEC_MAX_LOG_ADDRS); } |
| 6 7 10 5 10 10 10 10 10 1402 23 1 2 23 1399 10 10 10 10 2 10 6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 | // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ /* Copyright (c) 2008-2019, IBM Corporation */ #include <linux/init.h> #include <linux/errno.h> #include <linux/netdevice.h> #include <linux/inetdevice.h> #include <net/net_namespace.h> #include <linux/rtnetlink.h> #include <linux/if_arp.h> #include <linux/list.h> #include <linux/kernel.h> #include <linux/sched.h> #include <linux/module.h> #include <linux/dma-mapping.h> #include <net/addrconf.h> #include <rdma/ib_verbs.h> #include <rdma/ib_user_verbs.h> #include <rdma/rdma_netlink.h> #include <linux/kthread.h> #include "siw.h" #include "siw_verbs.h" MODULE_AUTHOR("Bernard Metzler"); MODULE_DESCRIPTION("Software iWARP Driver"); MODULE_LICENSE("Dual BSD/GPL"); /* transmit from user buffer, if possible */ const bool zcopy_tx = true; /* Restrict usage of GSO, if hardware peer iwarp is unable to process * large packets. try_gso = true lets siw try to use local GSO, * if peer agrees. Not using GSO severly limits siw maximum tx bandwidth. */ const bool try_gso; /* Attach siw also with loopback devices */ const bool loopback_enabled = true; /* We try to negotiate CRC on, if true */ const bool mpa_crc_required; /* MPA CRC on/off enforced */ const bool mpa_crc_strict; /* Control TCP_NODELAY socket option */ const bool siw_tcp_nagle; /* Select MPA version to be used during connection setup */ u_char mpa_version = MPA_REVISION_2; /* Selects MPA P2P mode (additional handshake during connection * setup, if true. */ const bool peer_to_peer; struct task_struct *siw_tx_thread[NR_CPUS]; static int siw_device_register(struct siw_device *sdev, const char *name) { struct ib_device *base_dev = &sdev->base_dev; static int dev_id = 1; int rv; sdev->vendor_part_id = dev_id++; rv = ib_register_device(base_dev, name, NULL); if (rv) { pr_warn("siw: device registration error %d\n", rv); return rv; } siw_dbg(base_dev, "HWaddr=%pM\n", sdev->raw_gid); return 0; } static void siw_device_cleanup(struct ib_device *base_dev) { struct siw_device *sdev = to_siw_dev(base_dev); xa_destroy(&sdev->qp_xa); xa_destroy(&sdev->mem_xa); } static int siw_dev_qualified(struct net_device *netdev) { /* * Additional hardware support can be added here * (e.g. ARPHRD_FDDI, ARPHRD_ATM, ...) - see * <linux/if_arp.h> for type identifiers. */ if (netdev->type == ARPHRD_ETHER || netdev->type == ARPHRD_IEEE802 || netdev->type == ARPHRD_NONE || (netdev->type == ARPHRD_LOOPBACK && loopback_enabled)) return 1; return 0; } static DEFINE_PER_CPU(atomic_t, siw_use_cnt); static struct { struct cpumask **tx_valid_cpus; int num_nodes; } siw_cpu_info; static void siw_destroy_cpulist(int number) { int i = 0; while (i < number) kfree(siw_cpu_info.tx_valid_cpus[i++]); kfree(siw_cpu_info.tx_valid_cpus); siw_cpu_info.tx_valid_cpus = NULL; } static int siw_init_cpulist(void) { int i, num_nodes = nr_node_ids; memset(siw_tx_thread, 0, sizeof(siw_tx_thread)); siw_cpu_info.num_nodes = num_nodes; siw_cpu_info.tx_valid_cpus = kzalloc_objs(struct cpumask *, num_nodes); if (!siw_cpu_info.tx_valid_cpus) { siw_cpu_info.num_nodes = 0; return -ENOMEM; } for (i = 0; i < siw_cpu_info.num_nodes; i++) { siw_cpu_info.tx_valid_cpus[i] = kzalloc_obj(struct cpumask); if (!siw_cpu_info.tx_valid_cpus[i]) goto out_err; cpumask_clear(siw_cpu_info.tx_valid_cpus[i]); } for_each_possible_cpu(i) cpumask_set_cpu(i, siw_cpu_info.tx_valid_cpus[cpu_to_node(i)]); return 0; out_err: siw_cpu_info.num_nodes = 0; siw_destroy_cpulist(i); return -ENOMEM; } /* * Choose CPU with least number of active QP's from NUMA node of * TX interface. */ int siw_get_tx_cpu(struct siw_device *sdev) { const struct cpumask *tx_cpumask; int i, num_cpus, cpu, min_use, node = sdev->numa_node, tx_cpu = -1; if (node < 0) tx_cpumask = cpu_online_mask; else tx_cpumask = siw_cpu_info.tx_valid_cpus[node]; num_cpus = cpumask_weight(tx_cpumask); if (!num_cpus) { /* no CPU on this NUMA node */ tx_cpumask = cpu_online_mask; num_cpus = cpumask_weight(tx_cpumask); } if (!num_cpus) goto out; cpu = cpumask_first(tx_cpumask); for (i = 0, min_use = SIW_MAX_QP; i < num_cpus; i++, cpu = cpumask_next(cpu, tx_cpumask)) { int usage; /* Skip any cores which have no TX thread */ if (!siw_tx_thread[cpu]) continue; usage = atomic_read(&per_cpu(siw_use_cnt, cpu)); if (usage <= min_use) { tx_cpu = cpu; min_use = usage; } } siw_dbg(&sdev->base_dev, "tx cpu %d, node %d, %d qp's\n", tx_cpu, node, min_use); out: if (tx_cpu >= 0) atomic_inc(&per_cpu(siw_use_cnt, tx_cpu)); else pr_warn("siw: no tx cpu found\n"); return tx_cpu; } void siw_put_tx_cpu(int cpu) { atomic_dec(&per_cpu(siw_use_cnt, cpu)); } static struct ib_qp *siw_get_base_qp(struct ib_device *base_dev, int id) { struct siw_qp *qp = siw_qp_id2obj(to_siw_dev(base_dev), id); if (qp) { /* * siw_qp_id2obj() increments object reference count */ siw_qp_put(qp); return &qp->base_qp; } return NULL; } static const struct ib_device_ops siw_device_ops = { .owner = THIS_MODULE, .uverbs_abi_ver = SIW_ABI_VERSION, .driver_id = RDMA_DRIVER_SIW, .alloc_mr = siw_alloc_mr, .alloc_pd = siw_alloc_pd, .alloc_ucontext = siw_alloc_ucontext, .create_cq = siw_create_cq, .create_qp = siw_create_qp, .create_srq = siw_create_srq, .dealloc_driver = siw_device_cleanup, .dealloc_pd = siw_dealloc_pd, .dealloc_ucontext = siw_dealloc_ucontext, .dereg_mr = siw_dereg_mr, .destroy_cq = siw_destroy_cq, .destroy_qp = siw_destroy_qp, .destroy_srq = siw_destroy_srq, .get_dma_mr = siw_get_dma_mr, .get_port_immutable = siw_get_port_immutable, .iw_accept = siw_accept, .iw_add_ref = siw_qp_get_ref, .iw_connect = siw_connect, .iw_create_listen = siw_create_listen, .iw_destroy_listen = siw_destroy_listen, .iw_get_qp = siw_get_base_qp, .iw_reject = siw_reject, .iw_rem_ref = siw_qp_put_ref, .map_mr_sg = siw_map_mr_sg, .mmap = siw_mmap, .mmap_free = siw_mmap_free, .modify_qp = siw_verbs_modify_qp, .modify_srq = siw_modify_srq, .poll_cq = siw_poll_cq, .post_recv = siw_post_receive, .post_send = siw_post_send, .post_srq_recv = siw_post_srq_recv, .query_device = siw_query_device, .query_gid = siw_query_gid, .query_port = siw_query_port, .query_qp = siw_query_qp, .query_srq = siw_query_srq, .req_notify_cq = siw_req_notify_cq, .reg_user_mr = siw_reg_user_mr, INIT_RDMA_OBJ_SIZE(ib_cq, siw_cq, base_cq), INIT_RDMA_OBJ_SIZE(ib_pd, siw_pd, base_pd), INIT_RDMA_OBJ_SIZE(ib_qp, siw_qp, base_qp), INIT_RDMA_OBJ_SIZE(ib_srq, siw_srq, base_srq), INIT_RDMA_OBJ_SIZE(ib_ucontext, siw_ucontext, base_ucontext), }; static struct siw_device *siw_device_create(struct net_device *netdev) { struct siw_device *sdev = NULL; struct ib_device *base_dev; int rv; sdev = ib_alloc_device(siw_device, base_dev); if (!sdev) return NULL; base_dev = &sdev->base_dev; if (netdev->addr_len) { memcpy(sdev->raw_gid, netdev->dev_addr, min_t(unsigned int, netdev->addr_len, ETH_ALEN)); } else { /* * This device does not have a HW address, but * connection mangagement requires a unique gid. */ eth_random_addr(sdev->raw_gid); } addrconf_addr_eui48((u8 *)&base_dev->node_guid, sdev->raw_gid); base_dev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POST_SEND); base_dev->node_type = RDMA_NODE_RNIC; memcpy(base_dev->node_desc, SIW_NODE_DESC_COMMON, sizeof(SIW_NODE_DESC_COMMON)); /* * Current model (one-to-one device association): * One Softiwarp device per net_device or, equivalently, * per physical port. */ base_dev->phys_port_cnt = 1; base_dev->num_comp_vectors = num_possible_cpus(); xa_init_flags(&sdev->qp_xa, XA_FLAGS_ALLOC1); xa_init_flags(&sdev->mem_xa, XA_FLAGS_ALLOC1); ib_set_device_ops(base_dev, &siw_device_ops); rv = ib_device_set_netdev(base_dev, netdev, 1); if (rv) goto error; memcpy(base_dev->iw_ifname, netdev->name, sizeof(base_dev->iw_ifname)); /* Disable TCP port mapping */ base_dev->iw_driver_flags = IW_F_NO_PORT_MAP; sdev->attrs.max_qp = SIW_MAX_QP; sdev->attrs.max_qp_wr = SIW_MAX_QP_WR; sdev->attrs.max_ord = SIW_MAX_ORD_QP; sdev->attrs.max_ird = SIW_MAX_IRD_QP; sdev->attrs.max_sge = SIW_MAX_SGE; sdev->attrs.max_sge_rd = SIW_MAX_SGE_RD; sdev->attrs.max_cq = SIW_MAX_CQ; sdev->attrs.max_cqe = SIW_MAX_CQE; sdev->attrs.max_mr = SIW_MAX_MR; sdev->attrs.max_pd = SIW_MAX_PD; sdev->attrs.max_mw = SIW_MAX_MW; sdev->attrs.max_srq = SIW_MAX_SRQ; sdev->attrs.max_srq_wr = SIW_MAX_SRQ_WR; sdev->attrs.max_srq_sge = SIW_MAX_SGE; INIT_LIST_HEAD(&sdev->cep_list); INIT_LIST_HEAD(&sdev->qp_list); atomic_set(&sdev->num_ctx, 0); atomic_set(&sdev->num_srq, 0); atomic_set(&sdev->num_qp, 0); atomic_set(&sdev->num_cq, 0); atomic_set(&sdev->num_mr, 0); atomic_set(&sdev->num_pd, 0); sdev->numa_node = dev_to_node(&netdev->dev); spin_lock_init(&sdev->lock); return sdev; error: ib_dealloc_device(base_dev); return NULL; } static int siw_netdev_event(struct notifier_block *nb, unsigned long event, void *arg) { struct net_device *netdev = netdev_notifier_info_to_dev(arg); struct ib_device *base_dev; struct siw_device *sdev; dev_dbg(&netdev->dev, "siw: event %lu\n", event); base_dev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_SIW); if (!base_dev) return NOTIFY_OK; sdev = to_siw_dev(base_dev); switch (event) { case NETDEV_REGISTER: /* * Device registration now handled only by * rdma netlink commands. So it shall be impossible * to end up here with a valid siw device. */ siw_dbg(base_dev, "unexpected NETDEV_REGISTER event\n"); break; case NETDEV_UNREGISTER: ib_unregister_device_queued(&sdev->base_dev); break; case NETDEV_CHANGEADDR: siw_port_event(sdev, 1, IB_EVENT_LID_CHANGE); break; /* * All other events are not handled */ default: break; } ib_device_put(&sdev->base_dev); return NOTIFY_OK; } static struct notifier_block siw_netdev_nb = { .notifier_call = siw_netdev_event, }; static int siw_newlink(const char *basedev_name, struct net_device *netdev) { struct ib_device *base_dev; struct siw_device *sdev = NULL; int rv = -ENOMEM; if (!siw_dev_qualified(netdev)) return -EINVAL; base_dev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_SIW); if (base_dev) { ib_device_put(base_dev); return -EEXIST; } sdev = siw_device_create(netdev); if (sdev) { dev_dbg(&netdev->dev, "siw: new device\n"); ib_mark_name_assigned_by_user(&sdev->base_dev); rv = siw_device_register(sdev, basedev_name); if (rv) ib_dealloc_device(&sdev->base_dev); } return rv; } static struct rdma_link_ops siw_link_ops = { .type = "siw", .newlink = siw_newlink, }; /* * siw_init_module - Initialize Softiwarp module and register with netdev * subsystem. */ static __init int siw_init_module(void) { int rv; if (SENDPAGE_THRESH < SIW_MAX_INLINE) { pr_info("siw: sendpage threshold too small: %u\n", (int)SENDPAGE_THRESH); rv = -EINVAL; goto out_error; } rv = siw_init_cpulist(); if (rv) goto out_error; rv = siw_cm_init(); if (rv) goto out_error; if (!siw_create_tx_threads()) { pr_info("siw: Could not start any TX thread\n"); rv = -ENOMEM; goto out_error; } rv = register_netdevice_notifier(&siw_netdev_nb); if (rv) goto out_error; rdma_link_register(&siw_link_ops); pr_info("SoftiWARP attached\n"); return 0; out_error: siw_stop_tx_threads(); pr_info("SoftIWARP attach failed. Error: %d\n", rv); siw_cm_exit(); siw_destroy_cpulist(siw_cpu_info.num_nodes); return rv; } static void __exit siw_exit_module(void) { siw_stop_tx_threads(); unregister_netdevice_notifier(&siw_netdev_nb); rdma_link_unregister(&siw_link_ops); ib_unregister_driver(RDMA_DRIVER_SIW); siw_cm_exit(); siw_destroy_cpulist(siw_cpu_info.num_nodes); pr_info("SoftiWARP detached\n"); } module_init(siw_init_module); module_exit(siw_exit_module); MODULE_ALIAS_RDMA_LINK("siw"); |
| 1 1 1 1 1 1 1 1 7 7 7 7 7 7 2 2 2 7 7 7 7 7 7 46 46 2 1 1 2 2 5 6 1 3 3 3 2 1 3 2 3 45 45 44 39 38 3 3 3 2 2 2 2 1 1 1 10 10 3 3 3 1 1 1 4 4 4 7 7 7 7 6 7 7 7 6 1 1 1 7 7 7 7 7 2 7 2 7 4 3 3 3 4 4 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 7 7 7 7 7 7 2 2 7 1 7 7 7 7 7 7 7 7 7 7 7 7 2 2 2 2 6 7 7 7 7 7 7 7 7 2 7 7 7 7 7 7 7 7 7 2 2 2 2 2 2 2 2 7 7 7 7 7 7 6 7 7 7 7 7 7 6 7 7 4 4 4 4 4 4 4 4 4 4 4 2 4 4 4 4 4 2 4 4 4 4 4 4 4 4 4 4 4 4 4 4 1 4 3 2 1 2 2 3 3 3 3 2 3 3 3 3 3 3 3 3 3 1 3 3 3 3 3 3 3 3 3 3 3 1 1 3 1 1 1 1 1 1 3 3 3 1 1 3 3 3 3 2 1 3 3 3 45 45 45 45 45 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 | // SPDX-License-Identifier: GPL-2.0-or-later /* */ #include <linux/init.h> #include <linux/slab.h> #include <linux/bitrev.h> #include <linux/ratelimit.h> #include <linux/usb.h> #include <linux/usb/audio.h> #include <linux/usb/audio-v2.h> #include <sound/core.h> #include <sound/pcm.h> #include <sound/pcm_params.h> #include "usbaudio.h" #include "card.h" #include "quirks.h" #include "endpoint.h" #include "helper.h" #include "pcm.h" #include "clock.h" #include "power.h" #include "media.h" #include "implicit.h" #define SUBSTREAM_FLAG_DATA_EP_STARTED 0 #define SUBSTREAM_FLAG_SYNC_EP_STARTED 1 /* return the estimated delay based on USB frame counters */ static snd_pcm_uframes_t snd_usb_pcm_delay(struct snd_usb_substream *subs, struct snd_pcm_runtime *runtime) { unsigned int current_frame_number; unsigned int frame_diff; int est_delay; int queued; if (subs->direction == SNDRV_PCM_STREAM_PLAYBACK) { queued = bytes_to_frames(runtime, subs->inflight_bytes); if (!queued) return 0; } else if (!subs->running) { return 0; } current_frame_number = usb_get_current_frame_number(subs->dev); /* * HCD implementations use different widths, use lower 8 bits. * The delay will be managed up to 256ms, which is more than * enough */ frame_diff = (current_frame_number - subs->last_frame_number) & 0xff; /* Approximation based on number of samples per USB frame (ms), some truncation for 44.1 but the estimate is good enough */ est_delay = frame_diff * runtime->rate / 1000; if (subs->direction == SNDRV_PCM_STREAM_PLAYBACK) { est_delay = queued - est_delay; if (est_delay < 0) est_delay = 0; } return est_delay; } /* * return the current pcm pointer. just based on the hwptr_done value. */ static snd_pcm_uframes_t snd_usb_pcm_pointer(struct snd_pcm_substream *substream) { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_usb_substream *subs = runtime->private_data; unsigned int hwptr_done; if (atomic_read(&subs->stream->chip->shutdown)) return SNDRV_PCM_POS_XRUN; scoped_guard(spinlock, &subs->lock) { hwptr_done = subs->hwptr_done; runtime->delay = snd_usb_pcm_delay(subs, runtime); } return bytes_to_frames(runtime, hwptr_done); } /* * find a matching audio format */ static const struct audioformat * find_format(struct list_head *fmt_list_head, snd_pcm_format_t format, unsigned int rate, unsigned int channels, bool strict_match, struct snd_usb_substream *subs) { const struct audioformat *fp; const struct audioformat *found = NULL; int cur_attr = 0, attr; list_for_each_entry(fp, fmt_list_head, list) { if (strict_match) { if (!(fp->formats & pcm_format_to_bits(format))) continue; if (fp->channels != channels) continue; } if (rate < fp->rate_min || rate > fp->rate_max) continue; if (!(fp->rates & SNDRV_PCM_RATE_CONTINUOUS)) { unsigned int i; for (i = 0; i < fp->nr_rates; i++) if (fp->rate_table[i] == rate) break; if (i >= fp->nr_rates) continue; } attr = fp->ep_attr & USB_ENDPOINT_SYNCTYPE; if (!found) { found = fp; cur_attr = attr; continue; } /* avoid async out and adaptive in if the other method * supports the same format. * this is a workaround for the case like * M-audio audiophile USB. */ if (subs && attr != cur_attr) { if ((attr == USB_ENDPOINT_SYNC_ASYNC && subs->direction == SNDRV_PCM_STREAM_PLAYBACK) || (attr == USB_ENDPOINT_SYNC_ADAPTIVE && subs->direction == SNDRV_PCM_STREAM_CAPTURE)) continue; if ((cur_attr == USB_ENDPOINT_SYNC_ASYNC && subs->direction == SNDRV_PCM_STREAM_PLAYBACK) || (cur_attr == USB_ENDPOINT_SYNC_ADAPTIVE && subs->direction == SNDRV_PCM_STREAM_CAPTURE)) { found = fp; cur_attr = attr; continue; } } /* find the format with the largest max. packet size */ if (fp->maxpacksize > found->maxpacksize) { found = fp; cur_attr = attr; } } return found; } const struct audioformat * snd_usb_find_format(struct list_head *fmt_list_head, snd_pcm_format_t format, unsigned int rate, unsigned int channels, bool strict_match, struct snd_usb_substream *subs) { return find_format(fmt_list_head, format, rate, channels, strict_match, subs); } EXPORT_SYMBOL_GPL(snd_usb_find_format); static const struct audioformat * find_substream_format(struct snd_usb_substream *subs, const struct snd_pcm_hw_params *params) { return find_format(&subs->fmt_list, params_format(params), params_rate(params), params_channels(params), true, subs); } const struct audioformat * snd_usb_find_substream_format(struct snd_usb_substream *subs, const struct snd_pcm_hw_params *params) { return find_substream_format(subs, params); } EXPORT_SYMBOL_GPL(snd_usb_find_substream_format); bool snd_usb_pcm_has_fixed_rate(struct snd_usb_substream *subs) { const struct audioformat *fp; struct snd_usb_audio *chip; int rate = -1; if (!subs) return false; chip = subs->stream->chip; if (!(chip->quirk_flags & QUIRK_FLAG_FIXED_RATE)) return false; list_for_each_entry(fp, &subs->fmt_list, list) { if (fp->rates & SNDRV_PCM_RATE_CONTINUOUS) return false; if (fp->nr_rates < 1) continue; if (fp->nr_rates > 1) return false; if (rate < 0) { rate = fp->rate_table[0]; continue; } if (rate != fp->rate_table[0]) return false; } return true; } static int init_pitch_v1(struct snd_usb_audio *chip, int ep) { struct usb_device *dev = chip->dev; unsigned char data[1]; int err; data[0] = 1; err = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), UAC_SET_CUR, USB_TYPE_CLASS|USB_RECIP_ENDPOINT|USB_DIR_OUT, UAC_EP_CS_ATTR_PITCH_CONTROL << 8, ep, data, sizeof(data)); return err; } static int init_pitch_v2(struct snd_usb_audio *chip, int ep) { struct usb_device *dev = chip->dev; unsigned char data[1]; int err; data[0] = 1; err = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), UAC2_CS_CUR, USB_TYPE_CLASS | USB_RECIP_ENDPOINT | USB_DIR_OUT, UAC2_EP_CS_PITCH << 8, 0, data, sizeof(data)); return err; } /* * initialize the pitch control and sample rate */ int snd_usb_init_pitch(struct snd_usb_audio *chip, const struct audioformat *fmt) { int err; /* if endpoint doesn't have pitch control, bail out */ if (!(fmt->attributes & UAC_EP_CS_ATTR_PITCH_CONTROL)) return 0; usb_audio_dbg(chip, "enable PITCH for EP 0x%x\n", fmt->endpoint); switch (fmt->protocol) { case UAC_VERSION_1: err = init_pitch_v1(chip, fmt->endpoint); break; case UAC_VERSION_2: err = init_pitch_v2(chip, fmt->endpoint); break; default: return 0; } if (err < 0) { usb_audio_err(chip, "failed to enable PITCH for EP 0x%x\n", fmt->endpoint); return err; } return 0; } static bool stop_endpoints(struct snd_usb_substream *subs, bool keep_pending) { bool stopped = 0; if (test_and_clear_bit(SUBSTREAM_FLAG_SYNC_EP_STARTED, &subs->flags)) { snd_usb_endpoint_stop(subs->sync_endpoint, keep_pending); stopped = true; } if (test_and_clear_bit(SUBSTREAM_FLAG_DATA_EP_STARTED, &subs->flags)) { snd_usb_endpoint_stop(subs->data_endpoint, keep_pending); stopped = true; } return stopped; } static int start_endpoints(struct snd_usb_substream *subs) { int err; if (!subs->data_endpoint) return -EINVAL; if (!test_and_set_bit(SUBSTREAM_FLAG_DATA_EP_STARTED, &subs->flags)) { err = snd_usb_endpoint_start(subs->data_endpoint); if (err < 0) { clear_bit(SUBSTREAM_FLAG_DATA_EP_STARTED, &subs->flags); goto error; } } if (subs->sync_endpoint && !test_and_set_bit(SUBSTREAM_FLAG_SYNC_EP_STARTED, &subs->flags)) { err = snd_usb_endpoint_start(subs->sync_endpoint); if (err < 0) { clear_bit(SUBSTREAM_FLAG_SYNC_EP_STARTED, &subs->flags); goto error; } } return 0; error: stop_endpoints(subs, false); return err; } static void sync_pending_stops(struct snd_usb_substream *subs) { snd_usb_endpoint_sync_pending_stop(subs->sync_endpoint); snd_usb_endpoint_sync_pending_stop(subs->data_endpoint); } /* PCM sync_stop callback */ static int snd_usb_pcm_sync_stop(struct snd_pcm_substream *substream) { struct snd_usb_substream *subs = substream->runtime->private_data; sync_pending_stops(subs); return 0; } /* Set up sync endpoint */ int snd_usb_audioformat_set_sync_ep(struct snd_usb_audio *chip, struct audioformat *fmt) { struct usb_device *dev = chip->dev; struct usb_host_interface *alts; struct usb_interface_descriptor *altsd; unsigned int ep, attr, sync_attr; bool is_playback; int err; if (fmt->sync_ep) return 0; /* already set up */ alts = snd_usb_get_host_interface(chip, fmt->iface, fmt->altsetting); if (!alts) return 0; altsd = get_iface_desc(alts); err = snd_usb_parse_implicit_fb_quirk(chip, fmt, alts); if (err > 0) return 0; /* matched */ /* * Generic sync EP handling */ if (fmt->ep_idx > 0 || altsd->bNumEndpoints < 2) return 0; is_playback = !(get_endpoint(alts, 0)->bEndpointAddress & USB_DIR_IN); attr = fmt->ep_attr & USB_ENDPOINT_SYNCTYPE; if ((is_playback && (attr == USB_ENDPOINT_SYNC_SYNC || attr == USB_ENDPOINT_SYNC_ADAPTIVE)) || (!is_playback && attr != USB_ENDPOINT_SYNC_ADAPTIVE)) return 0; sync_attr = get_endpoint(alts, 1)->bmAttributes; /* * In case of illegal SYNC_NONE for OUT endpoint, we keep going to see * if we don't find a sync endpoint, as on M-Audio Transit. In case of * error fall back to SYNC mode and don't create sync endpoint */ /* check sync-pipe endpoint */ /* ... and check descriptor size before accessing bSynchAddress because there is a version of the SB Audigy 2 NX firmware lacking the audio fields in the endpoint descriptors */ if ((sync_attr & USB_ENDPOINT_XFERTYPE_MASK) != USB_ENDPOINT_XFER_ISOC || (get_endpoint(alts, 1)->bLength >= USB_DT_ENDPOINT_AUDIO_SIZE && get_endpoint(alts, 1)->bSynchAddress != 0)) { dev_err(&dev->dev, "%d:%d : invalid sync pipe. bmAttributes %02x, bLength %d, bSynchAddress %02x\n", fmt->iface, fmt->altsetting, get_endpoint(alts, 1)->bmAttributes, get_endpoint(alts, 1)->bLength, get_endpoint(alts, 1)->bSynchAddress); if (is_playback && attr == USB_ENDPOINT_SYNC_NONE) return 0; return -EINVAL; } ep = get_endpoint(alts, 1)->bEndpointAddress; if (get_endpoint(alts, 0)->bLength >= USB_DT_ENDPOINT_AUDIO_SIZE && get_endpoint(alts, 0)->bSynchAddress != 0 && ((is_playback && ep != (unsigned int)(get_endpoint(alts, 0)->bSynchAddress | USB_DIR_IN)) || (!is_playback && ep != (unsigned int)(get_endpoint(alts, 0)->bSynchAddress & ~USB_DIR_IN)))) { dev_err(&dev->dev, "%d:%d : invalid sync pipe. is_playback %d, ep %02x, bSynchAddress %02x\n", fmt->iface, fmt->altsetting, is_playback, ep, get_endpoint(alts, 0)->bSynchAddress); if (is_playback && attr == USB_ENDPOINT_SYNC_NONE) return 0; return -EINVAL; } fmt->sync_ep = ep; fmt->sync_iface = altsd->bInterfaceNumber; fmt->sync_altsetting = altsd->bAlternateSetting; fmt->sync_ep_idx = 1; if ((sync_attr & USB_ENDPOINT_USAGE_MASK) == USB_ENDPOINT_USAGE_IMPLICIT_FB) fmt->implicit_fb = 1; dev_dbg(&dev->dev, "%d:%d: found sync_ep=0x%x, iface=%d, alt=%d, implicit_fb=%d\n", fmt->iface, fmt->altsetting, fmt->sync_ep, fmt->sync_iface, fmt->sync_altsetting, fmt->implicit_fb); return 0; } static int snd_usb_pcm_change_state(struct snd_usb_substream *subs, int state) { int ret; if (!subs->str_pd) return 0; ret = snd_usb_power_domain_set(subs->stream->chip, subs->str_pd, state); if (ret < 0) { dev_err(&subs->dev->dev, "Cannot change Power Domain ID: %d to state: %d. Err: %d\n", subs->str_pd->pd_id, state, ret); return ret; } return 0; } int snd_usb_pcm_suspend(struct snd_usb_stream *as) { int ret; ret = snd_usb_pcm_change_state(&as->substream[0], UAC3_PD_STATE_D2); if (ret < 0) return ret; ret = snd_usb_pcm_change_state(&as->substream[1], UAC3_PD_STATE_D2); if (ret < 0) return ret; return 0; } int snd_usb_pcm_resume(struct snd_usb_stream *as) { int ret; ret = snd_usb_pcm_change_state(&as->substream[0], UAC3_PD_STATE_D1); if (ret < 0) return ret; ret = snd_usb_pcm_change_state(&as->substream[1], UAC3_PD_STATE_D1); if (ret < 0) return ret; return 0; } static void close_endpoints(struct snd_usb_audio *chip, struct snd_usb_substream *subs) { if (subs->data_endpoint) { snd_usb_endpoint_set_sync(chip, subs->data_endpoint, NULL); snd_usb_endpoint_close(chip, subs->data_endpoint); subs->data_endpoint = NULL; } if (subs->sync_endpoint) { snd_usb_endpoint_close(chip, subs->sync_endpoint); subs->sync_endpoint = NULL; } } int snd_usb_hw_params(struct snd_usb_substream *subs, struct snd_pcm_hw_params *hw_params) { struct snd_usb_audio *chip = subs->stream->chip; const struct audioformat *fmt; const struct audioformat *sync_fmt; bool fixed_rate, sync_fixed_rate; int ret; ret = snd_media_start_pipeline(subs); if (ret) return ret; fixed_rate = snd_usb_pcm_has_fixed_rate(subs); fmt = find_substream_format(subs, hw_params); if (!fmt) { usb_audio_dbg(chip, "cannot find format: format=%s, rate=%d, channels=%d\n", snd_pcm_format_name(params_format(hw_params)), params_rate(hw_params), params_channels(hw_params)); ret = -EINVAL; goto stop_pipeline; } if (fmt->implicit_fb) { sync_fmt = snd_usb_find_implicit_fb_sync_format(chip, fmt, hw_params, !subs->direction, &sync_fixed_rate); if (!sync_fmt) { usb_audio_dbg(chip, "cannot find sync format: ep=0x%x, iface=%d:%d, format=%s, rate=%d, channels=%d\n", fmt->sync_ep, fmt->sync_iface, fmt->sync_altsetting, snd_pcm_format_name(params_format(hw_params)), params_rate(hw_params), params_channels(hw_params)); ret = -EINVAL; goto stop_pipeline; } } else { sync_fmt = fmt; sync_fixed_rate = fixed_rate; } ret = snd_usb_lock_shutdown(chip); if (ret < 0) goto stop_pipeline; ret = snd_usb_pcm_change_state(subs, UAC3_PD_STATE_D0); if (ret < 0) goto unlock; if (subs->data_endpoint) { if (snd_usb_endpoint_compatible(chip, subs->data_endpoint, fmt, hw_params)) goto unlock; if (stop_endpoints(subs, false)) sync_pending_stops(subs); close_endpoints(chip, subs); } subs->data_endpoint = snd_usb_endpoint_open(chip, fmt, hw_params, false, fixed_rate); if (!subs->data_endpoint) { ret = -EINVAL; goto unlock; } if (fmt->sync_ep) { subs->sync_endpoint = snd_usb_endpoint_open(chip, sync_fmt, hw_params, fmt == sync_fmt, sync_fixed_rate); if (!subs->sync_endpoint) { ret = -EINVAL; goto unlock; } snd_usb_endpoint_set_sync(chip, subs->data_endpoint, subs->sync_endpoint); } scoped_guard(mutex, &chip->mutex) { subs->cur_audiofmt = fmt; } if (!subs->data_endpoint->need_setup) goto unlock; if (subs->sync_endpoint) { ret = snd_usb_endpoint_set_params(chip, subs->sync_endpoint); if (ret < 0) goto unlock; } ret = snd_usb_endpoint_set_params(chip, subs->data_endpoint); unlock: if (ret < 0) close_endpoints(chip, subs); snd_usb_unlock_shutdown(chip); stop_pipeline: if (ret < 0) snd_media_stop_pipeline(subs); return ret; } EXPORT_SYMBOL_GPL(snd_usb_hw_params); /* * hw_params callback * * allocate a buffer and set the given audio format. * * so far we use a physically linear buffer although packetize transfer * doesn't need a continuous area. * if sg buffer is supported on the later version of alsa, we'll follow * that. */ static int snd_usb_pcm_hw_params(struct snd_pcm_substream *substream, struct snd_pcm_hw_params *hw_params) { struct snd_usb_substream *subs = substream->runtime->private_data; return snd_usb_hw_params(subs, hw_params); } int snd_usb_hw_free(struct snd_usb_substream *subs) { struct snd_usb_audio *chip = subs->stream->chip; snd_media_stop_pipeline(subs); scoped_guard(mutex, &chip->mutex) { subs->cur_audiofmt = NULL; } CLASS(snd_usb_lock, pm)(chip); if (!pm.err) { if (stop_endpoints(subs, false)) sync_pending_stops(subs); close_endpoints(chip, subs); } return 0; } EXPORT_SYMBOL_GPL(snd_usb_hw_free); /* * hw_free callback * * reset the audio format and release the buffer */ static int snd_usb_pcm_hw_free(struct snd_pcm_substream *substream) { struct snd_usb_substream *subs = substream->runtime->private_data; return snd_usb_hw_free(subs); } /* free-wheeling mode? (e.g. dmix) */ static int in_free_wheeling_mode(struct snd_pcm_runtime *runtime) { return runtime->stop_threshold > runtime->buffer_size; } /* check whether early start is needed for playback stream */ static int lowlatency_playback_available(struct snd_pcm_runtime *runtime, struct snd_usb_substream *subs) { struct snd_usb_audio *chip = subs->stream->chip; if (subs->direction == SNDRV_PCM_STREAM_CAPTURE) return false; /* disabled via module option? */ if (!chip->lowlatency) return false; if (in_free_wheeling_mode(runtime)) return false; /* implicit feedback mode has own operation mode */ if (snd_usb_endpoint_implicit_feedback_sink(subs->data_endpoint)) return false; return true; } /* * prepare callback * * only a few subtle things... */ static int snd_usb_pcm_prepare(struct snd_pcm_substream *substream) { struct snd_pcm_runtime *runtime = substream->runtime; struct snd_usb_substream *subs = runtime->private_data; struct snd_usb_audio *chip = subs->stream->chip; int retry = 0; int ret; CLASS(snd_usb_lock, pm)(chip); if (pm.err < 0) return pm.err; if (snd_BUG_ON(!subs->data_endpoint)) return -EIO; ret = snd_usb_pcm_change_state(subs, UAC3_PD_STATE_D0); if (ret < 0) return ret; again: if (subs->sync_endpoint) { ret = snd_usb_endpoint_prepare(chip, subs->sync_endpoint); if (ret < 0) return ret; } ret = snd_usb_endpoint_prepare(chip, subs->data_endpoint); if (ret < 0) return ret; else if (ret > 0) snd_usb_set_format_quirk(subs, subs->cur_audiofmt); ret = 0; /* reset the pointer */ subs->buffer_bytes = frames_to_bytes(runtime, runtime->buffer_size); subs->inflight_bytes = 0; subs->hwptr_done = 0; subs->transfer_done = 0; subs->last_frame_number = 0; subs->period_elapsed_pending = 0; runtime->delay = 0; subs->lowlatency_playback = lowlatency_playback_available(runtime, subs); if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK && !subs->lowlatency_playback) { ret = start_endpoints(subs); /* if XRUN happens at starting streams (possibly with implicit * fb case), restart again, but only try once. */ if (ret == -EPIPE && !retry++) { sync_pending_stops(subs); goto again; } } return ret; } /* * h/w constraints */ #ifdef HW_CONST_DEBUG #define hwc_debug(fmt, args...) pr_debug(fmt, ##args) #else #define hwc_debug(fmt, args...) do { } while(0) #endif static const struct snd_pcm_hardware snd_usb_hardware = { .info = SNDRV_PCM_INFO_MMAP | SNDRV_PCM_INFO_MMAP_VALID | SNDRV_PCM_INFO_BATCH | SNDRV_PCM_INFO_INTERLEAVED | SNDRV_PCM_INFO_BLOCK_TRANSFER | SNDRV_PCM_INFO_PAUSE, .channels_min = 1, .channels_max = 256, .buffer_bytes_max = INT_MAX, /* limited by BUFFER_TIME later */ .period_bytes_min = 64, .period_bytes_max = INT_MAX, /* limited by PERIOD_TIME later */ .periods_min = 2, .periods_max = 1024, }; static int hw_check_valid_format(struct snd_usb_substream *subs, struct snd_pcm_hw_params *params, const struct audioformat *fp) { struct snd_interval *it = hw_param_interval(params, SNDRV_PCM_HW_PARAM_RATE); struct snd_interval *ct = hw_param_interval(params, SNDRV_PCM_HW_PARAM_CHANNELS); struct snd_mask *fmts = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT); struct snd_interval *pt = hw_param_interval(params, SNDRV_PCM_HW_PARAM_PERIOD_TIME); struct snd_mask check_fmts; unsigned int ptime; /* check the format */ snd_mask_none(&check_fmts); check_fmts.bits[0] = (u32)fp->formats; check_fmts.bits[1] = (u32)(fp->formats >> 32); snd_mask_intersect(&check_fmts, fmts); if (snd_mask_empty(&check_fmts)) { hwc_debug(" > check: no supported format 0x%llx\n", fp->formats); return 0; } /* check the channels */ if (fp->channels < ct->min || fp->channels > ct->max) { hwc_debug(" > check: no valid channels %d (%d/%d)\n", fp->channels, ct->min, ct->max); return 0; } /* check the rate is within the range */ if (fp->rate_min > it->max || (fp->rate_min == it->max && it->openmax)) { hwc_debug(" > check: rate_min %d > max %d\n", fp->rate_min, it->max); return 0; } if (fp->rate_max < it->min || (fp->rate_max == it->min && it->openmin)) { hwc_debug(" > check: rate_max %d < min %d\n", fp->rate_max, it->min); return 0; } /* check whether the period time is >= the data packet interval */ if (subs->speed != USB_SPEED_FULL) { ptime = 125 * (1 << fp->datainterval); if (ptime > pt->max || (ptime == pt->max && pt->openmax)) { hwc_debug(" > check: ptime %u > max %u\n", ptime, pt->max); return 0; } } return 1; } static int apply_hw_params_minmax(struct snd_interval *it, unsigned int rmin, unsigned int rmax) { int changed; if (rmin > rmax) { hwc_debug(" --> get empty\n"); it->empty = 1; return -EINVAL; } changed = 0; if (it->min < rmin) { it->min = rmin; it->openmin = 0; changed = 1; } if (it->max > rmax) { it->max = rmax; it->openmax = 0; changed = 1; } if (snd_interval_checkempty(it)) { it->empty = 1; return -EINVAL; } hwc_debug(" --> (%d, %d) (changed = %d)\n", it->min, it->max, changed); return changed; } /* get the specified endpoint object that is being used by other streams * (i.e. the parameter is locked) */ static const struct snd_usb_endpoint * get_endpoint_in_use(struct snd_usb_audio *chip, int endpoint, const struct snd_usb_endpoint *ref_ep) { const struct snd_usb_endpoint *ep; ep = snd_usb_get_endpoint(chip, endpoint); if (ep && ep->cur_audiofmt && (ep != ref_ep || ep->opened > 1)) return ep; return NULL; } static int hw_rule_rate(struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule) { struct snd_usb_substream *subs = rule->private; struct snd_usb_audio *chip = subs->stream->chip; const struct snd_usb_endpoint *ep; const struct audioformat *fp; struct snd_interval *it = hw_param_interval(params, SNDRV_PCM_HW_PARAM_RATE); unsigned int rmin, rmax, r; int i; hwc_debug("hw_rule_rate: (%d,%d)\n", it->min, it->max); rmin = UINT_MAX; rmax = 0; list_for_each_entry(fp, &subs->fmt_list, list) { if (!hw_check_valid_format(subs, params, fp)) continue; ep = get_endpoint_in_use(chip, fp->endpoint, subs->data_endpoint); if (ep) { hwc_debug("rate limit %d for ep#%x\n", ep->cur_rate, fp->endpoint); rmin = min(rmin, ep->cur_rate); rmax = max(rmax, ep->cur_rate); continue; } if (fp->implicit_fb) { ep = get_endpoint_in_use(chip, fp->sync_ep, subs->sync_endpoint); if (ep) { hwc_debug("rate limit %d for sync_ep#%x\n", ep->cur_rate, fp->sync_ep); rmin = min(rmin, ep->cur_rate); rmax = max(rmax, ep->cur_rate); continue; } } r = snd_usb_endpoint_get_clock_rate(chip, fp->clock); if (r > 0) { if (!snd_interval_test(it, r)) continue; rmin = min(rmin, r); rmax = max(rmax, r); continue; } if (fp->rate_table && fp->nr_rates) { for (i = 0; i < fp->nr_rates; i++) { r = fp->rate_table[i]; if (!snd_interval_test(it, r)) continue; rmin = min(rmin, r); rmax = max(rmax, r); } } else { rmin = min(rmin, fp->rate_min); rmax = max(rmax, fp->rate_max); } } return apply_hw_params_minmax(it, rmin, rmax); } static int hw_rule_channels(struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule) { struct snd_usb_substream *subs = rule->private; const struct audioformat *fp; struct snd_interval *it = hw_param_interval(params, SNDRV_PCM_HW_PARAM_CHANNELS); unsigned int rmin, rmax; hwc_debug("hw_rule_channels: (%d,%d)\n", it->min, it->max); rmin = UINT_MAX; rmax = 0; list_for_each_entry(fp, &subs->fmt_list, list) { if (!hw_check_valid_format(subs, params, fp)) continue; rmin = min(rmin, fp->channels); rmax = max(rmax, fp->channels); } return apply_hw_params_minmax(it, rmin, rmax); } static int apply_hw_params_format_bits(struct snd_mask *fmt, u64 fbits) { u32 oldbits[2]; int changed; oldbits[0] = fmt->bits[0]; oldbits[1] = fmt->bits[1]; fmt->bits[0] &= (u32)fbits; fmt->bits[1] &= (u32)(fbits >> 32); if (!fmt->bits[0] && !fmt->bits[1]) { hwc_debug(" --> get empty\n"); return -EINVAL; } changed = (oldbits[0] != fmt->bits[0] || oldbits[1] != fmt->bits[1]); hwc_debug(" --> %x:%x (changed = %d)\n", fmt->bits[0], fmt->bits[1], changed); return changed; } static int hw_rule_format(struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule) { struct snd_usb_substream *subs = rule->private; struct snd_usb_audio *chip = subs->stream->chip; const struct snd_usb_endpoint *ep; const struct audioformat *fp; struct snd_mask *fmt = hw_param_mask(params, SNDRV_PCM_HW_PARAM_FORMAT); u64 fbits; hwc_debug("hw_rule_format: %x:%x\n", fmt->bits[0], fmt->bits[1]); fbits = 0; list_for_each_entry(fp, &subs->fmt_list, list) { if (!hw_check_valid_format(subs, params, fp)) continue; ep = get_endpoint_in_use(chip, fp->endpoint, subs->data_endpoint); if (ep) { hwc_debug("format limit %d for ep#%x\n", ep->cur_format, fp->endpoint); fbits |= pcm_format_to_bits(ep->cur_format); continue; } if (fp->implicit_fb) { ep = get_endpoint_in_use(chip, fp->sync_ep, subs->sync_endpoint); if (ep) { hwc_debug("format limit %d for sync_ep#%x\n", ep->cur_format, fp->sync_ep); fbits |= pcm_format_to_bits(ep->cur_format); continue; } } fbits |= fp->formats; } return apply_hw_params_format_bits(fmt, fbits); } static int hw_rule_period_time(struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule) { struct snd_usb_substream *subs = rule->private; const struct audioformat *fp; struct snd_interval *it; unsigned char min_datainterval; unsigned int pmin; it = hw_param_interval(params, SNDRV_PCM_HW_PARAM_PERIOD_TIME); hwc_debug("hw_rule_period_time: (%u,%u)\n", it->min, it->max); min_datainterval = 0xff; list_for_each_entry(fp, &subs->fmt_list, list) { if (!hw_check_valid_format(subs, params, fp)) continue; min_datainterval = min(min_datainterval, fp->datainterval); } if (min_datainterval == 0xff) { hwc_debug(" --> get empty\n"); it->empty = 1; return -EINVAL; } pmin = 125 * (1 << min_datainterval); return apply_hw_params_minmax(it, pmin, UINT_MAX); } /* additional hw constraints for implicit feedback mode */ static int hw_rule_period_size_implicit_fb(struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule) { struct snd_usb_substream *subs = rule->private; struct snd_usb_audio *chip = subs->stream->chip; const struct audioformat *fp; const struct snd_usb_endpoint *ep; struct snd_interval *it; unsigned int rmin, rmax; it = hw_param_interval(params, SNDRV_PCM_HW_PARAM_PERIOD_SIZE); hwc_debug("hw_rule_period_size: (%u,%u)\n", it->min, it->max); rmin = UINT_MAX; rmax = 0; list_for_each_entry(fp, &subs->fmt_list, list) { if (!hw_check_valid_format(subs, params, fp)) continue; ep = get_endpoint_in_use(chip, fp->endpoint, subs->data_endpoint); if (ep) { hwc_debug("period size limit %d for ep#%x\n", ep->cur_period_frames, fp->endpoint); rmin = min(rmin, ep->cur_period_frames); rmax = max(rmax, ep->cur_period_frames); continue; } if (fp->implicit_fb) { ep = get_endpoint_in_use(chip, fp->sync_ep, subs->sync_endpoint); if (ep) { hwc_debug("period size limit %d for sync_ep#%x\n", ep->cur_period_frames, fp->sync_ep); rmin = min(rmin, ep->cur_period_frames); rmax = max(rmax, ep->cur_period_frames); continue; } } } if (!rmax) return 0; /* no limit by implicit fb */ return apply_hw_params_minmax(it, rmin, rmax); } static int hw_rule_periods_implicit_fb(struct snd_pcm_hw_params *params, struct snd_pcm_hw_rule *rule) { struct snd_usb_substream *subs = rule->private; struct snd_usb_audio *chip = subs->stream->chip; const struct audioformat *fp; const struct snd_usb_endpoint *ep; struct snd_interval *it; unsigned int rmin, rmax; it = hw_param_interval(params, SNDRV_PCM_HW_PARAM_PERIODS); hwc_debug("hw_rule_periods: (%u,%u)\n", it->min, it->max); rmin = UINT_MAX; rmax = 0; list_for_each_entry(fp, &subs->fmt_list, list) { if (!hw_check_valid_format(subs, params, fp)) continue; ep = get_endpoint_in_use(chip, fp->endpoint, subs->data_endpoint); if (ep) { hwc_debug("periods limit %d for ep#%x\n", ep->cur_buffer_periods, fp->endpoint); rmin = min(rmin, ep->cur_buffer_periods); rmax = max(rmax, ep->cur_buffer_periods); continue; } if (fp->implicit_fb) { ep = get_endpoint_in_use(chip, fp->sync_ep, subs->sync_endpoint); if (ep) { hwc_debug("periods limit %d for sync_ep#%x\n", ep->cur_buffer_periods, fp->sync_ep); rmin = min(rmin, ep->cur_buffer_periods); rmax = max(rmax, ep->cur_buffer_periods); continue; } } } if (!rmax) return 0; /* no limit by implicit fb */ return apply_hw_params_minmax(it, rmin, rmax); } /* * set up the runtime hardware information. */ static int setup_hw_info(struct snd_pcm_runtime *runtime, struct snd_usb_substream *subs) { const struct audioformat *fp; unsigned int pt, ptmin; int param_period_time_if_needed = -1; int err; runtime->hw.formats = subs->formats; runtime->hw.rate_min = 0x7fffffff; runtime->hw.rate_max = 0; runtime->hw.channels_min = 256; runtime->hw.channels_max = 0; runtime->hw.rates = 0; ptmin = UINT_MAX; /* check min/max rates and channels */ list_for_each_entry(fp, &subs->fmt_list, list) { runtime->hw.rates |= fp->rates; if (runtime->hw.rate_min > fp->rate_min) runtime->hw.rate_min = fp->rate_min; if (runtime->hw.rate_max < fp->rate_max) runtime->hw.rate_max = fp->rate_max; if (runtime->hw.channels_min > fp->channels) runtime->hw.channels_min = fp->channels; if (runtime->hw.channels_max < fp->channels) runtime->hw.channels_max = fp->channels; if (fp->fmt_type == UAC_FORMAT_TYPE_II && fp->frame_size > 0) { /* FIXME: there might be more than one audio formats... */ runtime->hw.period_bytes_min = runtime->hw.period_bytes_max = fp->frame_size; } pt = 125 * (1 << fp->datainterval); ptmin = min(ptmin, pt); } param_period_time_if_needed = SNDRV_PCM_HW_PARAM_PERIOD_TIME; if (subs->speed == USB_SPEED_FULL) /* full speed devices have fixed data packet interval */ ptmin = 1000; if (ptmin == 1000) /* if period time doesn't go below 1 ms, no rules needed */ param_period_time_if_needed = -1; err = snd_pcm_hw_constraint_minmax(runtime, SNDRV_PCM_HW_PARAM_PERIOD_TIME, ptmin, UINT_MAX); if (err < 0) return err; err = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_RATE, hw_rule_rate, subs, SNDRV_PCM_HW_PARAM_RATE, SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_CHANNELS, param_period_time_if_needed, -1); if (err < 0) return err; err = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_CHANNELS, hw_rule_channels, subs, SNDRV_PCM_HW_PARAM_CHANNELS, SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_RATE, param_period_time_if_needed, -1); if (err < 0) return err; err = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_FORMAT, hw_rule_format, subs, SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_RATE, SNDRV_PCM_HW_PARAM_CHANNELS, param_period_time_if_needed, -1); if (err < 0) return err; if (param_period_time_if_needed >= 0) { err = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_PERIOD_TIME, hw_rule_period_time, subs, SNDRV_PCM_HW_PARAM_FORMAT, SNDRV_PCM_HW_PARAM_CHANNELS, SNDRV_PCM_HW_PARAM_RATE, -1); if (err < 0) return err; } /* set max period and buffer sizes for 1 and 2 seconds, respectively */ err = snd_pcm_hw_constraint_minmax(runtime, SNDRV_PCM_HW_PARAM_PERIOD_TIME, 0, 1000000); if (err < 0) return err; err = snd_pcm_hw_constraint_minmax(runtime, SNDRV_PCM_HW_PARAM_BUFFER_TIME, 0, 2000000); if (err < 0) return err; /* additional hw constraints for implicit fb */ err = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_PERIOD_SIZE, hw_rule_period_size_implicit_fb, subs, SNDRV_PCM_HW_PARAM_PERIOD_SIZE, -1); if (err < 0) return err; err = snd_pcm_hw_rule_add(runtime, 0, SNDRV_PCM_HW_PARAM_PERIODS, hw_rule_periods_implicit_fb, subs, SNDRV_PCM_HW_PARAM_PERIODS, -1); if (err < 0) return err; list_for_each_entry(fp, &subs->fmt_list, list) { if (fp->implicit_fb) { runtime->hw.info |= SNDRV_PCM_INFO_JOINT_DUPLEX; break; } } return 0; } static int snd_usb_pcm_open(struct snd_pcm_substream *substream) { int direction = substream->stream; struct snd_usb_stream *as = snd_pcm_substream_chip(substream); struct snd_pcm_runtime *runtime = substream->runtime; struct snd_usb_substream *subs = &as->substream[direction]; struct snd_usb_audio *chip = subs->stream->chip; int ret; scoped_guard(mutex, &chip->mutex) { if (subs->opened) return -EBUSY; subs->opened = 1; } runtime->hw = snd_usb_hardware; /* need an explicit sync to catch applptr update in low-latency mode */ if (direction == SNDRV_PCM_STREAM_PLAYBACK && as->chip->lowlatency) runtime->hw.info |= SNDRV_PCM_INFO_SYNC_APPLPTR; runtime->private_data = subs; subs->pcm_substream = substream; /* runtime PM is also done there */ /* initialize DSD/DOP context */ subs->dsd_dop.byte_idx = 0; subs->dsd_dop.channel = 0; subs->dsd_dop.marker = 1; ret = setup_hw_info(runtime, subs); if (ret < 0) goto err_open; ret = snd_usb_autoresume(subs->stream->chip); if (ret < 0) goto err_open; ret = snd_media_stream_init(subs, as->pcm, direction); if (ret < 0) goto err_resume; return 0; err_resume: snd_usb_autosuspend(subs->stream->chip); err_open: scoped_guard(mutex, &chip->mutex) { subs->opened = 0; } return ret; } static int snd_usb_pcm_close(struct snd_pcm_substream *substream) { int direction = substream->stream; struct snd_usb_stream *as = snd_pcm_substream_chip(substream); struct snd_usb_substream *subs = &as->substream[direction]; struct snd_usb_audio *chip = subs->stream->chip; int ret; snd_media_stop_pipeline(subs); { CLASS(snd_usb_lock, pm)(subs->stream->chip); if (pm.err) return pm.err; ret = snd_usb_pcm_change_state(subs, UAC3_PD_STATE_D1); if (ret < 0) return ret; } subs->pcm_substream = NULL; snd_usb_autosuspend(subs->stream->chip); scoped_guard(mutex, &chip->mutex) { subs->opened = 0; } return 0; } /* Since a URB can handle only a single linear buffer, we must use double * buffering when the data to be transferred overflows the buffer boundary. * To avoid inconsistencies when updating hwptr_done, we use double buffering * for all URBs. */ static void retire_capture_urb(struct snd_usb_substream *subs, struct urb *urb) { struct snd_pcm_runtime *runtime = subs->pcm_substream->runtime; unsigned int stride, frames, bytes, oldptr; int i, period_elapsed = 0; unsigned char *cp; int current_frame_number; /* read frame number here, update pointer in critical section */ current_frame_number = usb_get_current_frame_number(subs->dev); stride = runtime->frame_bits >> 3; for (i = 0; i < urb->number_of_packets; i++) { cp = (unsigned char *)urb->transfer_buffer + urb->iso_frame_desc[i].offset + subs->pkt_offset_adj; if (urb->iso_frame_desc[i].status) dev_dbg_ratelimited(&subs->dev->dev, "frame %d active: %d\n", i, urb->iso_frame_desc[i].status); bytes = urb->iso_frame_desc[i].actual_length; if (subs->stream_offset_adj > 0) { unsigned int adj = min(subs->stream_offset_adj, bytes); cp += adj; bytes -= adj; subs->stream_offset_adj -= adj; } frames = bytes / stride; if (!subs->txfr_quirk) bytes = frames * stride; if (bytes % (runtime->sample_bits >> 3) != 0) { int oldbytes = bytes; bytes = frames * stride; dev_warn_ratelimited(&subs->dev->dev, "Corrected urb data len. %d->%d\n", oldbytes, bytes); } /* update the current pointer */ scoped_guard(spinlock_irqsave, &subs->lock) { oldptr = subs->hwptr_done; subs->hwptr_done += bytes; if (subs->hwptr_done >= subs->buffer_bytes) subs->hwptr_done -= subs->buffer_bytes; frames = (bytes + (oldptr % stride)) / stride; subs->transfer_done += frames; if (subs->transfer_done >= runtime->period_size) { subs->transfer_done -= runtime->period_size; period_elapsed = 1; } /* realign last_frame_number */ subs->last_frame_number = current_frame_number; } /* copy a data chunk */ if (oldptr + bytes > subs->buffer_bytes) { unsigned int bytes1 = subs->buffer_bytes - oldptr; memcpy(runtime->dma_area + oldptr, cp, bytes1); memcpy(runtime->dma_area, cp + bytes1, bytes - bytes1); } else { memcpy(runtime->dma_area + oldptr, cp, bytes); } } if (period_elapsed) snd_pcm_period_elapsed(subs->pcm_substream); } static void urb_ctx_queue_advance(struct snd_usb_substream *subs, struct urb *urb, unsigned int bytes) { struct snd_urb_ctx *ctx = urb->context; ctx->queued += bytes; subs->inflight_bytes += bytes; subs->hwptr_done += bytes; if (subs->hwptr_done >= subs->buffer_bytes) subs->hwptr_done -= subs->buffer_bytes; } static inline void fill_playback_urb_dsd_dop(struct snd_usb_substream *subs, struct urb *urb, unsigned int bytes) { struct snd_pcm_runtime *runtime = subs->pcm_substream->runtime; unsigned int dst_idx = 0; unsigned int src_idx = subs->hwptr_done; unsigned int wrap = subs->buffer_bytes; u8 *dst = urb->transfer_buffer; u8 *src = runtime->dma_area; static const u8 marker[] = { 0x05, 0xfa }; unsigned int queued = 0; /* * The DSP DOP format defines a way to transport DSD samples over * normal PCM data endpoints. It requires stuffing of marker bytes * (0x05 and 0xfa, alternating per sample frame), and then expects * 2 additional bytes of actual payload. The whole frame is stored * LSB. * * Hence, for a stereo transport, the buffer layout looks like this, * where L refers to left channel samples and R to right. * * L1 L2 0x05 R1 R2 0x05 L3 L4 0xfa R3 R4 0xfa * L5 L6 0x05 R5 R6 0x05 L7 L8 0xfa R7 R8 0xfa * ..... * */ while (bytes--) { if (++subs->dsd_dop.byte_idx == 3) { /* frame boundary? */ dst[dst_idx++] = marker[subs->dsd_dop.marker]; src_idx += 2; subs->dsd_dop.byte_idx = 0; if (++subs->dsd_dop.channel % runtime->channels == 0) { /* alternate the marker */ subs->dsd_dop.marker++; subs->dsd_dop.marker %= ARRAY_SIZE(marker); subs->dsd_dop.channel = 0; } } else { /* stuff the DSD payload */ int idx = (src_idx + subs->dsd_dop.byte_idx - 1) % wrap; if (subs->cur_audiofmt->dsd_bitrev) dst[dst_idx++] = bitrev8(src[idx]); else dst[dst_idx++] = src[idx]; queued++; } } urb_ctx_queue_advance(subs, urb, queued); } /* copy bit-reversed bytes onto transfer buffer */ static void fill_playback_urb_dsd_bitrev(struct snd_usb_substream *subs, struct urb *urb, unsigned int bytes) { struct snd_pcm_runtime *runtime = subs->pcm_substream->runtime; const u8 *src = runtime->dma_area; u8 *buf = urb->transfer_buffer; int i, ofs = subs->hwptr_done; for (i = 0; i < bytes; i++) { *buf++ = bitrev8(src[ofs]); if (++ofs >= subs->buffer_bytes) ofs = 0; } urb_ctx_queue_advance(subs, urb, bytes); } static void copy_to_urb(struct snd_usb_substream *subs, struct urb *urb, int offset, int stride, unsigned int bytes) { struct snd_pcm_runtime *runtime = subs->pcm_substream->runtime; if (subs->hwptr_done + bytes > subs->buffer_bytes) { /* err, the transferred area goes over buffer boundary. */ unsigned int bytes1 = subs->buffer_bytes - subs->hwptr_done; memcpy(urb->transfer_buffer + offset, runtime->dma_area + subs->hwptr_done, bytes1); memcpy(urb->transfer_buffer + offset + bytes1, runtime->dma_area, bytes - bytes1); } else { memcpy(urb->transfer_buffer + offset, runtime->dma_area + subs->hwptr_done, bytes); } urb_ctx_queue_advance(subs, urb, bytes); } static unsigned int copy_to_urb_quirk(struct snd_usb_substream *subs, struct urb *urb, int stride, unsigned int bytes) { __le32 packet_length; int i; /* Put __le32 length descriptor at start of each packet. */ for (i = 0; i < urb->number_of_packets; i++) { unsigned int length = urb->iso_frame_desc[i].length; unsigned int offset = urb->iso_frame_desc[i].offset; packet_length = cpu_to_le32(length); offset += i * sizeof(packet_length); urb->iso_frame_desc[i].offset = offset; urb->iso_frame_desc[i].length += sizeof(packet_length); memcpy(urb->transfer_buffer + offset, &packet_length, sizeof(packet_length)); copy_to_urb(subs, urb, offset + sizeof(packet_length), stride, length); } /* Adjust transfer size accordingly. */ bytes += urb->number_of_packets * sizeof(packet_length); return bytes; } static int prepare_playback_urb(struct snd_usb_substream *subs, struct urb *urb, bool in_stream_lock) { struct snd_pcm_runtime *runtime = subs->pcm_substream->runtime; struct snd_usb_endpoint *ep = subs->data_endpoint; struct snd_urb_ctx *ctx = urb->context; unsigned int frames, bytes; int counts; unsigned int transfer_done, frame_limit, avail = 0; int i, stride, period_elapsed = 0; stride = ep->stride; frames = 0; ctx->queued = 0; urb->number_of_packets = 0; scoped_guard(spinlock_irqsave, &subs->lock) { frame_limit = subs->frame_limit + ep->max_urb_frames; transfer_done = subs->transfer_done; if (subs->lowlatency_playback && runtime->state != SNDRV_PCM_STATE_DRAINING) { unsigned int hwptr = subs->hwptr_done / stride; /* calculate the byte offset-in-buffer of the appl_ptr */ avail = (runtime->control->appl_ptr - runtime->hw_ptr_base) % runtime->buffer_size; if (avail <= hwptr) avail += runtime->buffer_size; avail -= hwptr; } for (i = 0; i < ctx->packets; i++) { counts = snd_usb_endpoint_next_packet_size(ep, ctx, i, avail); if (counts < 0 || (frames + counts) * stride > ctx->buffer_size) break; /* set up descriptor */ urb->iso_frame_desc[i].offset = frames * stride; urb->iso_frame_desc[i].length = counts * stride; frames += counts; avail -= counts; urb->number_of_packets++; transfer_done += counts; if (transfer_done >= runtime->period_size) { transfer_done -= runtime->period_size; frame_limit = 0; period_elapsed = 1; if (subs->fmt_type == UAC_FORMAT_TYPE_II) { if (transfer_done > 0) { /* FIXME: fill-max mode is not * supported yet */ frames -= transfer_done; counts -= transfer_done; urb->iso_frame_desc[i].length = counts * stride; transfer_done = 0; } i++; if (i < ctx->packets) { /* add a transfer delimiter */ urb->iso_frame_desc[i].offset = frames * stride; urb->iso_frame_desc[i].length = 0; urb->number_of_packets++; } break; } } /* finish at the period boundary or after enough frames */ if ((period_elapsed || transfer_done >= frame_limit) && !snd_usb_endpoint_implicit_feedback_sink(ep)) break; } if (!frames) return -EAGAIN; bytes = frames * stride; subs->transfer_done = transfer_done; subs->frame_limit = frame_limit; if (unlikely(ep->cur_format == SNDRV_PCM_FORMAT_DSD_U16_LE && subs->cur_audiofmt->dsd_dop)) { fill_playback_urb_dsd_dop(subs, urb, bytes); } else if (unlikely(ep->cur_format == SNDRV_PCM_FORMAT_DSD_U8 && subs->cur_audiofmt->dsd_bitrev)) { fill_playback_urb_dsd_bitrev(subs, urb, bytes); } else { /* usual PCM */ if (!subs->tx_length_quirk) copy_to_urb(subs, urb, 0, stride, bytes); else bytes = copy_to_urb_quirk(subs, urb, stride, bytes); /* bytes is now amount of outgoing data */ } subs->last_frame_number = usb_get_current_frame_number(subs->dev); if (subs->trigger_tstamp_pending_update) { /* this is the first actual URB submitted, * update trigger timestamp to reflect actual start time */ snd_pcm_gettime(runtime, &runtime->trigger_tstamp); subs->trigger_tstamp_pending_update = false; } if (period_elapsed && !subs->running && subs->lowlatency_playback) { subs->period_elapsed_pending = 1; period_elapsed = 0; } } urb->transfer_buffer_length = bytes; if (period_elapsed) { if (in_stream_lock) snd_pcm_period_elapsed_under_stream_lock(subs->pcm_substream); else snd_pcm_period_elapsed(subs->pcm_substream); } return 0; } /* * process after playback data complete * - decrease the delay count again */ static void retire_playback_urb(struct snd_usb_substream *subs, struct urb *urb) { struct snd_urb_ctx *ctx = urb->context; bool period_elapsed = false; scoped_guard(spinlock_irqsave, &subs->lock) { if (ctx->queued) { if (subs->inflight_bytes >= ctx->queued) subs->inflight_bytes -= ctx->queued; else subs->inflight_bytes = 0; } subs->last_frame_number = usb_get_current_frame_number(subs->dev); if (subs->running) { period_elapsed = subs->period_elapsed_pending; subs->period_elapsed_pending = 0; } } if (period_elapsed) snd_pcm_period_elapsed(subs->pcm_substream); } /* PCM ack callback for the playback stream; * this plays a role only when the stream is running in low-latency mode. */ static int snd_usb_pcm_playback_ack(struct snd_pcm_substream *substream) { struct snd_usb_substream *subs = substream->runtime->private_data; struct snd_usb_endpoint *ep; if (!subs->lowlatency_playback || !subs->running) return 0; ep = subs->data_endpoint; if (!ep) return 0; /* When no more in-flight URBs available, try to process the pending * outputs here */ if (!ep->active_mask) return snd_usb_queue_pending_output_urbs(ep, true); return 0; } static int snd_usb_substream_playback_trigger(struct snd_pcm_substream *substream, int cmd) { struct snd_usb_substream *subs = substream->runtime->private_data; int err; switch (cmd) { case SNDRV_PCM_TRIGGER_START: subs->trigger_tstamp_pending_update = true; fallthrough; case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: snd_usb_endpoint_set_callback(subs->data_endpoint, prepare_playback_urb, retire_playback_urb, subs); if (subs->lowlatency_playback && cmd == SNDRV_PCM_TRIGGER_START) { if (in_free_wheeling_mode(substream->runtime)) subs->lowlatency_playback = false; err = start_endpoints(subs); if (err < 0) { snd_usb_endpoint_set_callback(subs->data_endpoint, NULL, NULL, NULL); return err; } } subs->running = 1; dev_dbg(&subs->dev->dev, "%d:%d Start Playback PCM\n", subs->cur_audiofmt->iface, subs->cur_audiofmt->altsetting); return 0; case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_STOP: stop_endpoints(subs, substream->runtime->state == SNDRV_PCM_STATE_DRAINING); snd_usb_endpoint_set_callback(subs->data_endpoint, NULL, NULL, NULL); subs->running = 0; dev_dbg(&subs->dev->dev, "%d:%d Stop Playback PCM\n", subs->cur_audiofmt->iface, subs->cur_audiofmt->altsetting); return 0; case SNDRV_PCM_TRIGGER_PAUSE_PUSH: /* keep retire_data_urb for delay calculation */ snd_usb_endpoint_set_callback(subs->data_endpoint, NULL, retire_playback_urb, subs); subs->running = 0; dev_dbg(&subs->dev->dev, "%d:%d Pause Playback PCM\n", subs->cur_audiofmt->iface, subs->cur_audiofmt->altsetting); return 0; } return -EINVAL; } static int snd_usb_substream_capture_trigger(struct snd_pcm_substream *substream, int cmd) { int err; struct snd_usb_substream *subs = substream->runtime->private_data; switch (cmd) { case SNDRV_PCM_TRIGGER_START: err = start_endpoints(subs); if (err < 0) return err; fallthrough; case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: snd_usb_endpoint_set_callback(subs->data_endpoint, NULL, retire_capture_urb, subs); subs->last_frame_number = usb_get_current_frame_number(subs->dev); subs->running = 1; dev_dbg(&subs->dev->dev, "%d:%d Start Capture PCM\n", subs->cur_audiofmt->iface, subs->cur_audiofmt->altsetting); return 0; case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_STOP: stop_endpoints(subs, false); fallthrough; case SNDRV_PCM_TRIGGER_PAUSE_PUSH: snd_usb_endpoint_set_callback(subs->data_endpoint, NULL, NULL, NULL); subs->running = 0; dev_dbg(&subs->dev->dev, "%d:%d Stop Capture PCM\n", subs->cur_audiofmt->iface, subs->cur_audiofmt->altsetting); return 0; } return -EINVAL; } static const struct snd_pcm_ops snd_usb_playback_ops = { .open = snd_usb_pcm_open, .close = snd_usb_pcm_close, .hw_params = snd_usb_pcm_hw_params, .hw_free = snd_usb_pcm_hw_free, .prepare = snd_usb_pcm_prepare, .trigger = snd_usb_substream_playback_trigger, .sync_stop = snd_usb_pcm_sync_stop, .pointer = snd_usb_pcm_pointer, .ack = snd_usb_pcm_playback_ack, }; static const struct snd_pcm_ops snd_usb_capture_ops = { .open = snd_usb_pcm_open, .close = snd_usb_pcm_close, .hw_params = snd_usb_pcm_hw_params, .hw_free = snd_usb_pcm_hw_free, .prepare = snd_usb_pcm_prepare, .trigger = snd_usb_substream_capture_trigger, .sync_stop = snd_usb_pcm_sync_stop, .pointer = snd_usb_pcm_pointer, }; void snd_usb_set_pcm_ops(struct snd_pcm *pcm, int stream) { const struct snd_pcm_ops *ops; ops = stream == SNDRV_PCM_STREAM_PLAYBACK ? &snd_usb_playback_ops : &snd_usb_capture_ops; snd_pcm_set_ops(pcm, stream, ops); } void snd_usb_preallocate_buffer(struct snd_usb_substream *subs) { struct snd_pcm *pcm = subs->stream->pcm; struct snd_pcm_substream *s = pcm->streams[subs->direction].substream; struct device *dev = subs->dev->bus->sysdev; if (snd_usb_use_vmalloc) snd_pcm_set_managed_buffer(s, SNDRV_DMA_TYPE_VMALLOC, NULL, 0, 0); else snd_pcm_set_managed_buffer(s, SNDRV_DMA_TYPE_DEV_SG, dev, 64*1024, 512*1024); } |
| 9 9 4 4 9 15 6 6 6 6 2 6 6 6 6 5 5 5 5 5 6 6 5 5 6 15 6 15 15 13 13 13 13 9 13 1 10 2 15 15 15 15 2 1 2 2 1 11 2 2 2 2 15 15 16 1 1 12 15 15 14 13 13 14 15 15 15 15 15 15 10 15 12 2 12 14 11 11 2 14 16 3 3 2 2 2 2 2 2 2 1 1 3 2 1 2 2 2 2 2 3 20 20 17 20 20 20 1 18 3 16 1 15 2 16 1 15 19 17 19 12 13 12 1 1 12 1 17 17 20 20 14 20 19 12 3 16 16 20 6 20 20 3 3 3 2 2 1 3 3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 | // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2018-2019 HUAWEI, Inc. * https://www.huawei.com/ */ #include "internal.h" #include <linux/unaligned.h> #include <trace/events/erofs.h> struct z_erofs_maprecorder { struct inode *inode; struct erofs_map_blocks *map; u64 lcn; /* compression extent information gathered */ u8 type, headtype; u16 clusterofs; u16 delta[2]; erofs_blk_t pblk; erofs_off_t nextpackoff; int compressedblks; bool partialref, in_mbox; }; static int z_erofs_load_full_lcluster(struct z_erofs_maprecorder *m, u64 lcn) { struct inode *const inode = m->inode; struct erofs_inode *const vi = EROFS_I(inode); const erofs_off_t pos = Z_EROFS_FULL_INDEX_START(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize) + lcn * sizeof(struct z_erofs_lcluster_index); struct z_erofs_lcluster_index *di; unsigned int advise; di = erofs_read_metabuf(&m->map->buf, inode->i_sb, pos, m->in_mbox); if (IS_ERR(di)) return PTR_ERR(di); m->lcn = lcn; m->nextpackoff = pos + sizeof(struct z_erofs_lcluster_index); advise = le16_to_cpu(di->di_advise); m->type = advise & Z_EROFS_LI_LCLUSTER_TYPE_MASK; if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { m->clusterofs = 1 << vi->z_lclusterbits; m->delta[0] = le16_to_cpu(di->di_u.delta[0]); if (m->delta[0] & Z_EROFS_LI_D0_CBLKCNT) { if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | Z_EROFS_ADVISE_BIG_PCLUSTER_2))) { DBG_BUGON(1); return -EFSCORRUPTED; } m->compressedblks = m->delta[0] & ~Z_EROFS_LI_D0_CBLKCNT; m->delta[0] = 1; } m->delta[1] = le16_to_cpu(di->di_u.delta[1]); } else { m->partialref = !!(advise & Z_EROFS_LI_PARTIAL_REF); m->clusterofs = le16_to_cpu(di->di_clusterofs); if (advise & Z_EROFS_LI_HOLE) { m->compressedblks = 0; m->pblk = EROFS_NULL_ADDR; } else { m->pblk = le32_to_cpu(di->di_u.blkaddr); } } return 0; } static unsigned int decode_compactedbits(unsigned int lobits, u8 *in, unsigned int pos, u8 *type) { const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7); const unsigned int lo = v & ((1 << lobits) - 1); *type = (v >> lobits) & 3; return lo; } static int get_compacted_la_distance(unsigned int lobits, unsigned int encodebits, unsigned int vcnt, u8 *in, int i) { unsigned int lo, d1 = 0; u8 type; DBG_BUGON(i >= vcnt); do { lo = decode_compactedbits(lobits, in, encodebits * i, &type); if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) return d1; ++d1; } while (++i < vcnt); /* vcnt - 1 (Z_EROFS_LCLUSTER_TYPE_NONHEAD) item */ if (!(lo & Z_EROFS_LI_D0_CBLKCNT)) d1 += lo - 1; return d1; } static int z_erofs_load_compact_lcluster(struct z_erofs_maprecorder *m, u64 lcn, bool lookahead) { struct inode *const inode = m->inode; struct erofs_inode *const vi = EROFS_I(inode); const erofs_off_t ebase = Z_EROFS_MAP_HEADER_END(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize); const unsigned int lclusterbits = vi->z_lclusterbits; const unsigned int totalidx = erofs_iblks(inode); unsigned int compacted_4b_initial, compacted_2b, amortizedshift; unsigned int vcnt, lo, lobits, encodebits, nblk, bytes; bool big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; erofs_off_t pos; u8 *in, type; int i; if (lcn >= totalidx || lclusterbits > 14) return -EINVAL; m->lcn = lcn; /* used to align to 32-byte (compacted_2b) alignment */ compacted_4b_initial = ((32 - ebase % 32) / 4) & 7; compacted_2b = 0; if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) && compacted_4b_initial < totalidx) compacted_2b = rounddown(totalidx - compacted_4b_initial, 16); pos = ebase; amortizedshift = 2; /* compact_4b */ if (lcn >= compacted_4b_initial) { pos += compacted_4b_initial * 4; lcn -= compacted_4b_initial; if (lcn < compacted_2b) { amortizedshift = 1; } else { pos += compacted_2b * 2; lcn -= compacted_2b; } } pos += lcn * (1 << amortizedshift); /* figure out the lcluster count in this pack */ if (1 << amortizedshift == 4 && lclusterbits <= 14) vcnt = 2; else if (1 << amortizedshift == 2 && lclusterbits <= 12) vcnt = 16; else return -EOPNOTSUPP; in = erofs_read_metabuf(&m->map->buf, inode->i_sb, pos, m->in_mbox); if (IS_ERR(in)) return PTR_ERR(in); /* it doesn't equal to round_up(..) */ m->nextpackoff = round_down(pos, vcnt << amortizedshift) + (vcnt << amortizedshift); lobits = max(lclusterbits, ilog2(Z_EROFS_LI_D0_CBLKCNT) + 1U); encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt; bytes = pos & ((vcnt << amortizedshift) - 1); in -= bytes; i = bytes >> amortizedshift; lo = decode_compactedbits(lobits, in, encodebits * i, &type); m->type = type; if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { m->clusterofs = 1 << lclusterbits; /* figure out lookahead_distance: delta[1] if needed */ if (lookahead) m->delta[1] = get_compacted_la_distance(lobits, encodebits, vcnt, in, i); if (lo & Z_EROFS_LI_D0_CBLKCNT) { if (!big_pcluster) { DBG_BUGON(1); return -EFSCORRUPTED; } m->compressedblks = lo & ~Z_EROFS_LI_D0_CBLKCNT; m->delta[0] = 1; return 0; } else if (i + 1 != (int)vcnt) { m->delta[0] = lo; return 0; } /* * since the last lcluster in the pack is special, * of which lo saves delta[1] rather than delta[0]. * Hence, get delta[0] by the previous lcluster indirectly. */ lo = decode_compactedbits(lobits, in, encodebits * (i - 1), &type); if (type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) lo = 0; else if (lo & Z_EROFS_LI_D0_CBLKCNT) lo = 1; m->delta[0] = lo + 1; return 0; } m->clusterofs = lo; m->delta[0] = 0; /* figout out blkaddr (pblk) for HEAD lclusters */ if (!big_pcluster) { nblk = 1; while (i > 0) { --i; lo = decode_compactedbits(lobits, in, encodebits * i, &type); if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) i -= lo; if (i >= 0) ++nblk; } } else { nblk = 0; while (i > 0) { --i; lo = decode_compactedbits(lobits, in, encodebits * i, &type); if (type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { if (lo & Z_EROFS_LI_D0_CBLKCNT) { --i; nblk += lo & ~Z_EROFS_LI_D0_CBLKCNT; continue; } /* bigpcluster shouldn't have plain d0 == 1 */ if (lo <= 1) { DBG_BUGON(1); return -EFSCORRUPTED; } i -= lo - 2; continue; } ++nblk; } } in += (vcnt << amortizedshift) - sizeof(__le32); m->pblk = le32_to_cpu(*(__le32 *)in) + nblk; return 0; } static int z_erofs_load_lcluster_from_disk(struct z_erofs_maprecorder *m, u64 lcn, bool lookahead) { struct erofs_inode *vi = EROFS_I(m->inode); int err; if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT) { err = z_erofs_load_compact_lcluster(m, lcn, lookahead); } else { DBG_BUGON(vi->datalayout != EROFS_INODE_COMPRESSED_FULL); err = z_erofs_load_full_lcluster(m, lcn); } if (err) return err; if (m->type >= Z_EROFS_LCLUSTER_TYPE_MAX) { erofs_err(m->inode->i_sb, "unknown type %u @ lcn %llu of nid %llu", m->type, lcn, EROFS_I(m->inode)->nid); DBG_BUGON(1); return -EOPNOTSUPP; } else if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && m->clusterofs >= (1 << vi->z_lclusterbits)) { DBG_BUGON(1); return -EFSCORRUPTED; } return 0; } static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m, unsigned int lookback_distance) { struct super_block *sb = m->inode->i_sb; struct erofs_inode *const vi = EROFS_I(m->inode); const unsigned int lclusterbits = vi->z_lclusterbits; while (m->lcn >= lookback_distance) { u64 lcn = m->lcn - lookback_distance; int err; if (!lookback_distance) break; err = z_erofs_load_lcluster_from_disk(m, lcn, false); if (err) return err; if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { lookback_distance = m->delta[0]; continue; } m->headtype = m->type; m->map->m_la = (lcn << lclusterbits) | m->clusterofs; return 0; } erofs_err(sb, "bogus lookback distance %u @ lcn %llu of nid %llu", lookback_distance, m->lcn, vi->nid); DBG_BUGON(1); return -EFSCORRUPTED; } static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m, unsigned int initial_lcn) { struct inode *inode = m->inode; struct super_block *sb = inode->i_sb; struct erofs_inode *vi = EROFS_I(inode); bool bigpcl1 = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1; bool bigpcl2 = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2; u64 lcn = m->lcn + 1; int err; DBG_BUGON(m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD); DBG_BUGON(m->type != m->headtype); if ((m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD1 && !bigpcl1) || ((m->headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN || m->headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) && !bigpcl2) || (lcn << vi->z_lclusterbits) >= inode->i_size) if (m->compressedblks < 0) m->compressedblks = 1; if (m->compressedblks >= 0) goto out; err = z_erofs_load_lcluster_from_disk(m, lcn, false); if (err) return err; /* * If the 1st NONHEAD lcluster has already been handled initially w/o * valid compressedblks, which means at least it mustn't be CBLKCNT, or * an internal implemenatation error is detected. * * The following code can also handle it properly anyway, but let's * BUG_ON in the debugging mode only for developers to notice that. */ DBG_BUGON(lcn == initial_lcn && m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD); if (m->type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) { /* * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type * rather than CBLKCNT, it's a 1 block-sized pcluster. */ if (m->compressedblks < 0) m->compressedblks = 1; } else if (m->delta[0] != 1 || m->compressedblks < 0) { erofs_err(sb, "bogus CBLKCNT @ lcn %llu of nid %llu", lcn, vi->nid); DBG_BUGON(1); return -EFSCORRUPTED; } out: if (!m->compressedblks) m->map->m_flags &= ~EROFS_MAP_MAPPED; m->map->m_plen = erofs_pos(sb, m->compressedblks); return 0; } static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m) { struct inode *inode = m->inode; struct erofs_inode *vi = EROFS_I(inode); struct erofs_map_blocks *map = m->map; unsigned int lclusterbits = vi->z_lclusterbits; u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits; int err; while (1) { /* handle the last EOF pcluster (no next HEAD lcluster) */ if ((lcn << lclusterbits) >= inode->i_size) { map->m_llen = inode->i_size - map->m_la; return 0; } err = z_erofs_load_lcluster_from_disk(m, lcn, true); if (err) return err; if (m->type == Z_EROFS_LCLUSTER_TYPE_NONHEAD) { /* work around invalid d1 generated by pre-1.0 mkfs */ if (unlikely(!m->delta[1])) { m->delta[1] = 1; DBG_BUGON(1); } } else if (m->type < Z_EROFS_LCLUSTER_TYPE_MAX) { if (lcn != headlcn) break; /* ends at the next HEAD lcluster */ m->delta[1] = 1; } lcn += m->delta[1]; } map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la; return 0; } static int z_erofs_map_blocks_fo(struct inode *inode, struct erofs_map_blocks *map, int flags) { struct erofs_inode *vi = EROFS_I(inode); struct super_block *sb = inode->i_sb; bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; bool ztailpacking = vi->z_idata_size; unsigned int lclusterbits = vi->z_lclusterbits; struct z_erofs_maprecorder m = { .inode = inode, .map = map, .in_mbox = erofs_inode_in_metabox(inode), .compressedblks = -1, }; unsigned int endoff; unsigned long initial_lcn; unsigned long long ofs, end; int err; ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la; if (fragment && !(flags & EROFS_GET_BLOCKS_FINDTAIL) && !vi->z_tailextent_headlcn) { map->m_la = 0; map->m_llen = inode->i_size; map->m_flags = EROFS_MAP_FRAGMENT; return 0; } initial_lcn = ofs >> lclusterbits; endoff = ofs & ((1 << lclusterbits) - 1); err = z_erofs_load_lcluster_from_disk(&m, initial_lcn, false); if (err) goto unmap_out; if ((flags & EROFS_GET_BLOCKS_FINDTAIL) && ztailpacking) vi->z_fragmentoff = m.nextpackoff; map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_PARTIAL_MAPPED; end = (m.lcn + 1ULL) << lclusterbits; if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD && endoff >= m.clusterofs) { m.headtype = m.type; map->m_la = (m.lcn << lclusterbits) | m.clusterofs; /* * For ztailpacking files, in order to inline data more * effectively, special EOF lclusters are now supported * which can have three parts at most. */ if (ztailpacking && end > inode->i_size) end = inode->i_size; } else { if (m.type != Z_EROFS_LCLUSTER_TYPE_NONHEAD) { end = (m.lcn << lclusterbits) | m.clusterofs; map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; m.delta[0] = 1; } /* get the corresponding first chunk */ err = z_erofs_extent_lookback(&m, m.delta[0]); if (err) goto unmap_out; } if (m.partialref) map->m_flags |= EROFS_MAP_PARTIAL_REF; map->m_llen = end - map->m_la; if (flags & EROFS_GET_BLOCKS_FINDTAIL) { vi->z_tailextent_headlcn = m.lcn; /* for non-compact indexes, fragmentoff is 64 bits */ if (fragment && vi->datalayout == EROFS_INODE_COMPRESSED_FULL) vi->z_fragmentoff |= (u64)m.pblk << 32; } if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) { map->m_flags |= EROFS_MAP_META; map->m_pa = vi->z_fragmentoff; map->m_plen = vi->z_idata_size; if (erofs_blkoff(sb, map->m_pa) + map->m_plen > sb->s_blocksize) { erofs_err(sb, "ztailpacking inline data across blocks @ nid %llu", vi->nid); err = -EFSCORRUPTED; goto unmap_out; } } else if (fragment && m.lcn == vi->z_tailextent_headlcn) { map->m_flags = EROFS_MAP_FRAGMENT; } else { map->m_pa = erofs_pos(sb, m.pblk); err = z_erofs_get_extent_compressedlen(&m, initial_lcn); if (err) goto unmap_out; } if (m.headtype == Z_EROFS_LCLUSTER_TYPE_PLAIN) { if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; } else if (m.headtype == Z_EROFS_LCLUSTER_TYPE_HEAD2) { map->m_algorithmformat = vi->z_algorithmtype[1]; } else { map->m_algorithmformat = vi->z_algorithmtype[0]; } if ((flags & EROFS_GET_BLOCKS_FIEMAP) || ((flags & EROFS_GET_BLOCKS_READMORE) && (map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA || map->m_algorithmformat == Z_EROFS_COMPRESSION_DEFLATE || map->m_algorithmformat == Z_EROFS_COMPRESSION_ZSTD) && map->m_llen >= i_blocksize(inode))) { err = z_erofs_get_extent_decompressedlen(&m); if (!err) map->m_flags &= ~EROFS_MAP_PARTIAL_MAPPED; } unmap_out: erofs_unmap_metabuf(&m.map->buf); return err; } static int z_erofs_map_blocks_ext(struct inode *inode, struct erofs_map_blocks *map, int flags) { struct erofs_inode *vi = EROFS_I(inode); struct super_block *sb = inode->i_sb; bool interlaced = vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER; unsigned int recsz = z_erofs_extent_recsize(vi->z_advise); erofs_off_t pos = round_up(Z_EROFS_MAP_HEADER_END(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize), recsz); unsigned int bmask = sb->s_blocksize - 1; bool in_mbox = erofs_inode_in_metabox(inode); erofs_off_t lend = inode->i_size; erofs_off_t l, r, mid, pa, la, lstart; struct z_erofs_extent *ext; unsigned int fmt; bool last; map->m_flags = 0; if (recsz <= offsetof(struct z_erofs_extent, pstart_hi)) { if (recsz <= offsetof(struct z_erofs_extent, pstart_lo)) { ext = erofs_read_metabuf(&map->buf, sb, pos, in_mbox); if (IS_ERR(ext)) return PTR_ERR(ext); pa = le64_to_cpu(*(__le64 *)ext); pos += sizeof(__le64); lstart = 0; } else { lstart = round_down(map->m_la, 1 << vi->z_lclusterbits); pos += (lstart >> vi->z_lclusterbits) * recsz; pa = EROFS_NULL_ADDR; } for (; lstart <= map->m_la; lstart += 1 << vi->z_lclusterbits) { ext = erofs_read_metabuf(&map->buf, sb, pos, in_mbox); if (IS_ERR(ext)) return PTR_ERR(ext); map->m_plen = le32_to_cpu(ext->plen); if (pa != EROFS_NULL_ADDR) { map->m_pa = pa; pa += map->m_plen & Z_EROFS_EXTENT_PLEN_MASK; } else { map->m_pa = le32_to_cpu(ext->pstart_lo); } pos += recsz; } last = (lstart >= round_up(lend, 1 << vi->z_lclusterbits)); lend = min(lstart, lend); lstart -= 1 << vi->z_lclusterbits; } else { lstart = lend; for (l = 0, r = vi->z_extents; l < r; ) { mid = l + (r - l) / 2; ext = erofs_read_metabuf(&map->buf, sb, pos + mid * recsz, in_mbox); if (IS_ERR(ext)) return PTR_ERR(ext); la = le32_to_cpu(ext->lstart_lo); pa = le32_to_cpu(ext->pstart_lo) | (u64)le32_to_cpu(ext->pstart_hi) << 32; if (recsz > offsetof(struct z_erofs_extent, lstart_hi)) la |= (u64)le32_to_cpu(ext->lstart_hi) << 32; if (la > map->m_la) { r = mid; if (la > lend) { DBG_BUGON(1); return -EFSCORRUPTED; } lend = la; } else { l = mid + 1; if (map->m_la == la) r = min(l + 1, r); lstart = la; map->m_plen = le32_to_cpu(ext->plen); map->m_pa = pa; } } last = (l >= vi->z_extents); } if (lstart < lend) { map->m_la = lstart; if (last && (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER)) { map->m_flags = EROFS_MAP_FRAGMENT; vi->z_fragmentoff = map->m_plen; if (recsz > offsetof(struct z_erofs_extent, pstart_lo)) vi->z_fragmentoff |= map->m_pa << 32; } else if (map->m_plen & Z_EROFS_EXTENT_PLEN_MASK) { map->m_flags |= EROFS_MAP_MAPPED; fmt = map->m_plen >> Z_EROFS_EXTENT_PLEN_FMT_BIT; if (map->m_plen & Z_EROFS_EXTENT_PLEN_PARTIAL) map->m_flags |= EROFS_MAP_PARTIAL_REF; map->m_plen &= Z_EROFS_EXTENT_PLEN_MASK; if (fmt) map->m_algorithmformat = fmt - 1; else if (interlaced && !((map->m_pa | map->m_plen) & bmask)) map->m_algorithmformat = Z_EROFS_COMPRESSION_INTERLACED; else map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED; } } map->m_llen = lend - map->m_la; return 0; } static int z_erofs_fill_inode(struct inode *inode, struct erofs_map_blocks *map) { struct erofs_inode *const vi = EROFS_I(inode); struct super_block *const sb = inode->i_sb; struct z_erofs_map_header *h; erofs_off_t pos; int err = 0; if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) { /* * paired with smp_mb() at the end of the function to ensure * fields will only be observed after the bit is set. */ smp_mb(); return 0; } if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE)) return -ERESTARTSYS; if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) goto out_unlock; pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8); h = erofs_read_metabuf(&map->buf, sb, pos, erofs_inode_in_metabox(inode)); if (IS_ERR(h)) { err = PTR_ERR(h); goto out_unlock; } /* * if the highest bit of the 8-byte map header is set, the whole file * is stored in the packed inode. The rest bits keeps z_fragmentoff. */ if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) { vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER; vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63); vi->z_tailextent_headlcn = 0; goto done; } vi->z_advise = le16_to_cpu(h->h_advise); vi->z_lclusterbits = sb->s_blocksize_bits + (h->h_clusterbits & 15); if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && (vi->z_advise & Z_EROFS_ADVISE_EXTENTS)) { vi->z_extents = le32_to_cpu(h->h_extents_lo) | ((u64)le16_to_cpu(h->h_extents_hi) << 32); goto done; } vi->z_algorithmtype[0] = h->h_algorithmtype & 15; vi->z_algorithmtype[1] = h->h_algorithmtype >> 4; if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff); else if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) vi->z_idata_size = le16_to_cpu(h->h_idata_size); if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) && vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 | Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu", vi->nid); err = -EFSCORRUPTED; goto out_unlock; } if (vi->datalayout == EROFS_INODE_COMPRESSED_COMPACT && !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^ !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) { erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu", vi->nid); err = -EFSCORRUPTED; goto out_unlock; } if (vi->z_idata_size || (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER)) { struct erofs_map_blocks tm = { .buf = __EROFS_BUF_INITIALIZER }; err = z_erofs_map_blocks_fo(inode, &tm, EROFS_GET_BLOCKS_FINDTAIL); erofs_put_metabuf(&tm.buf); if (err < 0) goto out_unlock; } done: /* paired with smp_mb() at the beginning of the function */ smp_mb(); set_bit(EROFS_I_Z_INITED_BIT, &vi->flags); out_unlock: clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags); return err; } static int z_erofs_map_sanity_check(struct inode *inode, struct erofs_map_blocks *map) { struct erofs_sb_info *sbi = EROFS_I_SB(inode); u64 pend; if (!(map->m_flags & EROFS_MAP_MAPPED)) return 0; if (unlikely(map->m_algorithmformat >= Z_EROFS_COMPRESSION_RUNTIME_MAX)) { erofs_err(inode->i_sb, "unknown algorithm %d @ pos %llu for nid %llu, please upgrade kernel", map->m_algorithmformat, map->m_la, EROFS_I(inode)->nid); return -EOPNOTSUPP; } if (map->m_algorithmformat < Z_EROFS_COMPRESSION_MAX) { if (!(sbi->available_compr_algs & BIT(map->m_algorithmformat))) { erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu", map->m_algorithmformat, EROFS_I(inode)->nid); return -EFSCORRUPTED; } if (EROFS_MAP_FULL(map->m_flags) && map->m_llen < map->m_plen && map->m_la + map->m_llen < inode->i_size) { erofs_err(inode->i_sb, "too much compressed data @ la %llu of nid %llu", map->m_la, EROFS_I(inode)->nid); return -EFSCORRUPTED; } } else if (map->m_llen > map->m_plen) { erofs_err(inode->i_sb, "not enough plain data on disk @ la %llu of nid %llu", map->m_la, EROFS_I(inode)->nid); return -EFSCORRUPTED; } if (unlikely(map->m_plen > Z_EROFS_PCLUSTER_MAX_SIZE || map->m_llen > Z_EROFS_PCLUSTER_MAX_DSIZE)) return -EOPNOTSUPP; /* Filesystems beyond 48-bit physical block addresses are invalid */ if (unlikely(check_add_overflow(map->m_pa, map->m_plen, &pend) || (pend >> sbi->blkszbits) >= BIT_ULL(48))) return -EFSCORRUPTED; return 0; } int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map, int flags) { struct erofs_inode *const vi = EROFS_I(inode); int err = 0; trace_erofs_map_blocks_enter(inode, map, flags); if (map->m_la >= inode->i_size) { /* post-EOF unmapped extent */ map->m_llen = map->m_la + 1 - inode->i_size; map->m_la = inode->i_size; map->m_flags = 0; } else { err = z_erofs_fill_inode(inode, map); if (!err) { if (vi->datalayout == EROFS_INODE_COMPRESSED_FULL && (vi->z_advise & Z_EROFS_ADVISE_EXTENTS)) err = z_erofs_map_blocks_ext(inode, map, flags); else err = z_erofs_map_blocks_fo(inode, map, flags); } if (!err) err = z_erofs_map_sanity_check(inode, map); if (err) map->m_llen = 0; } trace_erofs_map_blocks_exit(inode, map, flags, err); return err; } static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset, loff_t length, unsigned int flags, struct iomap *iomap, struct iomap *srcmap) { int ret; struct erofs_map_blocks map = { .m_la = offset }; ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP); erofs_put_metabuf(&map.buf); if (ret < 0) return ret; iomap->bdev = inode->i_sb->s_bdev; iomap->offset = map.m_la; iomap->length = map.m_llen; if (map.m_flags & EROFS_MAP_FRAGMENT) { iomap->type = IOMAP_MAPPED; iomap->addr = IOMAP_NULL_ADDR; } else if (map.m_flags & EROFS_MAP_MAPPED) { iomap->type = IOMAP_MAPPED; iomap->addr = map.m_pa; } else { iomap->type = IOMAP_HOLE; iomap->addr = IOMAP_NULL_ADDR; /* * No strict rule on how to describe extents for post EOF, yet * we need to do like below. Otherwise, iomap itself will get * into an endless loop on post EOF. * * Calculate the effective offset by subtracting extent start * (map.m_la) from the requested offset, and add it to length. * (NB: offset >= map.m_la always) */ if (iomap->offset >= inode->i_size) iomap->length = length + offset - map.m_la; } iomap->flags = 0; return 0; } const struct iomap_ops z_erofs_iomap_report_ops = { .iomap_begin = z_erofs_iomap_begin_report, }; |
| 226 228 228 227 227 226 228 228 227 226 226 243 241 2 238 241 243 115 115 114 113 3 113 92 91 91 428 427 427 427 428 53 423 173 302 75 7 7 7 7 420 22 418 418 417 27 418 159 266 420 21 2 419 21 419 417 8 1 345 339 8 8 338 344 214 18 46 220 214 245 213 3 2 212 211 212 209 208 138 7 206 207 206 207 157 207 69 205 20 201 70 200 132 201 203 205 213 152 149 9 140 152 192 193 154 152 340 323 319 11 12 327 327 34 6 326 324 291 292 301 301 253 199 75 192 253 149 151 142 197 160 51 51 1 229 33 33 33 33 3 33 38 17 21 21 39 6 3 4 2 37 35 25 35 39 1162 1168 196 1163 847 843 851 1 6 6 6 145 129 144 145 152 152 19 13 |