| 2 11 7 2 1 16 16 16 1 4 5 6 9 5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 | // SPDX-License-Identifier: GPL-2.0-only /* Xtables module to match packets using a BPF filter. * Copyright 2013 Google Inc. * Written by Willem de Bruijn <willemb@google.com> */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/module.h> #include <linux/syscalls.h> #include <linux/skbuff.h> #include <linux/filter.h> #include <linux/bpf.h> #include <linux/netfilter/xt_bpf.h> #include <linux/netfilter/x_tables.h> MODULE_AUTHOR("Willem de Bruijn <willemb@google.com>"); MODULE_DESCRIPTION("Xtables: BPF filter match"); MODULE_LICENSE("GPL"); MODULE_ALIAS("ipt_bpf"); MODULE_ALIAS("ip6t_bpf"); static int __bpf_mt_check_bytecode(struct sock_filter *insns, __u16 len, struct bpf_prog **ret) { struct sock_fprog_kern program; if (len > XT_BPF_MAX_NUM_INSTR) return -EINVAL; program.len = len; program.filter = insns; if (bpf_prog_create(ret, &program)) { pr_info_ratelimited("check failed: parse error\n"); return -EINVAL; } return 0; } static int __bpf_mt_check_fd(int fd, struct bpf_prog **ret) { struct bpf_prog *prog; prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER); if (IS_ERR(prog)) return PTR_ERR(prog); *ret = prog; return 0; } static int __bpf_mt_check_path(const char *path, struct bpf_prog **ret) { if (strnlen(path, XT_BPF_PATH_MAX) == XT_BPF_PATH_MAX) return -EINVAL; *ret = bpf_prog_get_type_path(path, BPF_PROG_TYPE_SOCKET_FILTER); return PTR_ERR_OR_ZERO(*ret); } static int bpf_mt_check(const struct xt_mtchk_param *par) { struct xt_bpf_info *info = par->matchinfo; return __bpf_mt_check_bytecode(info->bpf_program, info->bpf_program_num_elem, &info->filter); } static int bpf_mt_check_v1(const struct xt_mtchk_param *par) { struct xt_bpf_info_v1 *info = par->matchinfo; if (info->mode == XT_BPF_MODE_BYTECODE) return __bpf_mt_check_bytecode(info->bpf_program, info->bpf_program_num_elem, &info->filter); else if (info->mode == XT_BPF_MODE_FD_ELF) return __bpf_mt_check_fd(info->fd, &info->filter); else if (info->mode == XT_BPF_MODE_PATH_PINNED) return __bpf_mt_check_path(info->path, &info->filter); else return -EINVAL; } static bool bpf_mt(const struct sk_buff *skb, struct xt_action_param *par) { const struct xt_bpf_info *info = par->matchinfo; return bpf_prog_run(info->filter, skb); } static bool bpf_mt_v1(const struct sk_buff *skb, struct xt_action_param *par) { const struct xt_bpf_info_v1 *info = par->matchinfo; return !!bpf_prog_run_save_cb(info->filter, (struct sk_buff *) skb); } static void bpf_mt_destroy(const struct xt_mtdtor_param *par) { const struct xt_bpf_info *info = par->matchinfo; bpf_prog_destroy(info->filter); } static void bpf_mt_destroy_v1(const struct xt_mtdtor_param *par) { const struct xt_bpf_info_v1 *info = par->matchinfo; bpf_prog_destroy(info->filter); } static struct xt_match bpf_mt_reg[] __read_mostly = { { .name = "bpf", .revision = 0, .family = NFPROTO_UNSPEC, .checkentry = bpf_mt_check, .match = bpf_mt, .destroy = bpf_mt_destroy, .matchsize = sizeof(struct xt_bpf_info), .usersize = offsetof(struct xt_bpf_info, filter), .me = THIS_MODULE, }, { .name = "bpf", .revision = 1, .family = NFPROTO_UNSPEC, .checkentry = bpf_mt_check_v1, .match = bpf_mt_v1, .destroy = bpf_mt_destroy_v1, .matchsize = sizeof(struct xt_bpf_info_v1), .usersize = offsetof(struct xt_bpf_info_v1, filter), .me = THIS_MODULE, }, }; static int __init bpf_mt_init(void) { return xt_register_matches(bpf_mt_reg, ARRAY_SIZE(bpf_mt_reg)); } static void __exit bpf_mt_exit(void) { xt_unregister_matches(bpf_mt_reg, ARRAY_SIZE(bpf_mt_reg)); } module_init(bpf_mt_init); module_exit(bpf_mt_exit); |
| 9620 2102 8 10 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 | /* SPDX-License-Identifier: GPL-2.0 */ #undef TRACE_SYSTEM #define TRACE_SYSTEM percpu #if !defined(_TRACE_PERCPU_H) || defined(TRACE_HEADER_MULTI_READ) #define _TRACE_PERCPU_H #include <linux/tracepoint.h> #include <trace/events/mmflags.h> TRACE_EVENT(percpu_alloc_percpu, TP_PROTO(unsigned long call_site, bool reserved, bool is_atomic, size_t size, size_t align, void *base_addr, int off, void __percpu *ptr, size_t bytes_alloc, gfp_t gfp_flags), TP_ARGS(call_site, reserved, is_atomic, size, align, base_addr, off, ptr, bytes_alloc, gfp_flags), TP_STRUCT__entry( __field( unsigned long, call_site ) __field( bool, reserved ) __field( bool, is_atomic ) __field( size_t, size ) __field( size_t, align ) __field( void *, base_addr ) __field( int, off ) __field( void __percpu *, ptr ) __field( size_t, bytes_alloc ) __field( unsigned long, gfp_flags ) ), TP_fast_assign( __entry->call_site = call_site; __entry->reserved = reserved; __entry->is_atomic = is_atomic; __entry->size = size; __entry->align = align; __entry->base_addr = base_addr; __entry->off = off; __entry->ptr = ptr; __entry->bytes_alloc = bytes_alloc; __entry->gfp_flags = (__force unsigned long)gfp_flags; ), TP_printk("call_site=%pS reserved=%d is_atomic=%d size=%zu align=%zu base_addr=%p off=%d ptr=%p bytes_alloc=%zu gfp_flags=%s", (void *)__entry->call_site, __entry->reserved, __entry->is_atomic, __entry->size, __entry->align, __entry->base_addr, __entry->off, __entry->ptr, __entry->bytes_alloc, show_gfp_flags(__entry->gfp_flags)) ); TRACE_EVENT(percpu_free_percpu, TP_PROTO(void *base_addr, int off, void __percpu *ptr), TP_ARGS(base_addr, off, ptr), TP_STRUCT__entry( __field( void *, base_addr ) __field( int, off ) __field( void __percpu *, ptr ) ), TP_fast_assign( __entry->base_addr = base_addr; __entry->off = off; __entry->ptr = ptr; ), TP_printk("base_addr=%p off=%d ptr=%p", __entry->base_addr, __entry->off, __entry->ptr) ); TRACE_EVENT(percpu_alloc_percpu_fail, TP_PROTO(bool reserved, bool is_atomic, size_t size, size_t align), TP_ARGS(reserved, is_atomic, size, align), TP_STRUCT__entry( __field( bool, reserved ) __field( bool, is_atomic ) __field( size_t, size ) __field( size_t, align ) ), TP_fast_assign( __entry->reserved = reserved; __entry->is_atomic = is_atomic; __entry->size = size; __entry->align = align; ), TP_printk("reserved=%d is_atomic=%d size=%zu align=%zu", __entry->reserved, __entry->is_atomic, __entry->size, __entry->align) ); TRACE_EVENT(percpu_create_chunk, TP_PROTO(void *base_addr), TP_ARGS(base_addr), TP_STRUCT__entry( __field( void *, base_addr ) ), TP_fast_assign( __entry->base_addr = base_addr; ), TP_printk("base_addr=%p", __entry->base_addr) ); TRACE_EVENT(percpu_destroy_chunk, TP_PROTO(void *base_addr), TP_ARGS(base_addr), TP_STRUCT__entry( __field( void *, base_addr ) ), TP_fast_assign( __entry->base_addr = base_addr; ), TP_printk("base_addr=%p", __entry->base_addr) ); #endif /* _TRACE_PERCPU_H */ #include <trace/define_trace.h> |
| 38 2 35 1 1 1 1 32 32 26 6 2 10 8 2 1 1 2 49 8 13 11 25 15 5 10 14 1 14 10 4 10 6 6 4 2 2 2 1 1 2 5 2 3 2 1 3 3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 | // SPDX-License-Identifier: GPL-2.0 /* * linux/fs/ext4/resize.c * * Support for resizing an ext4 filesystem while it is mounted. * * Copyright (C) 2001, 2002 Andreas Dilger <adilger@clusterfs.com> * * This could probably be made into a module, because it is not often in use. */ #include <linux/errno.h> #include <linux/slab.h> #include <linux/jiffies.h> #include "ext4_jbd2.h" struct ext4_rcu_ptr { struct rcu_head rcu; void *ptr; }; static void ext4_rcu_ptr_callback(struct rcu_head *head) { struct ext4_rcu_ptr *ptr; ptr = container_of(head, struct ext4_rcu_ptr, rcu); kvfree(ptr->ptr); kfree(ptr); } void ext4_kvfree_array_rcu(void *to_free) { struct ext4_rcu_ptr *ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); if (ptr) { ptr->ptr = to_free; call_rcu(&ptr->rcu, ext4_rcu_ptr_callback); return; } synchronize_rcu(); kvfree(to_free); } int ext4_resize_begin(struct super_block *sb) { struct ext4_sb_info *sbi = EXT4_SB(sb); int ret = 0; if (!capable(CAP_SYS_RESOURCE)) return -EPERM; /* * If the reserved GDT blocks is non-zero, the resize_inode feature * should always be set. */ if (sbi->s_es->s_reserved_gdt_blocks && !ext4_has_feature_resize_inode(sb)) { ext4_error(sb, "resize_inode disabled but reserved GDT blocks non-zero"); return -EFSCORRUPTED; } /* * If we are not using the primary superblock/GDT copy don't resize, * because the user tools have no way of handling this. Probably a * bad time to do it anyways. */ if (EXT4_B2C(sbi, sbi->s_sbh->b_blocknr) != le32_to_cpu(sbi->s_es->s_first_data_block)) { ext4_warning(sb, "won't resize using backup superblock at %llu", (unsigned long long)sbi->s_sbh->b_blocknr); return -EPERM; } /* * We are not allowed to do online-resizing on a filesystem mounted * with error, because it can destroy the filesystem easily. */ if (sbi->s_mount_state & EXT4_ERROR_FS) { ext4_warning(sb, "There are errors in the filesystem, " "so online resizing is not allowed"); return -EPERM; } if (ext4_has_feature_sparse_super2(sb)) { ext4_msg(sb, KERN_ERR, "Online resizing not supported with sparse_super2"); return -EOPNOTSUPP; } if (test_and_set_bit_lock(EXT4_FLAGS_RESIZING, &sbi->s_ext4_flags)) ret = -EBUSY; return ret; } int ext4_resize_end(struct super_block *sb, bool update_backups) { clear_bit_unlock(EXT4_FLAGS_RESIZING, &EXT4_SB(sb)->s_ext4_flags); smp_mb__after_atomic(); if (update_backups) return ext4_update_overhead(sb, true); return 0; } static ext4_grpblk_t ext4_group_overhead_blocks(struct super_block *sb, ext4_group_t group) { ext4_grpblk_t overhead; overhead = ext4_bg_num_gdb(sb, group); if (ext4_bg_has_super(sb, group)) overhead += 1 + le16_to_cpu(EXT4_SB(sb)->s_es->s_reserved_gdt_blocks); return overhead; } #define outside(b, first, last) ((b) < (first) || (b) >= (last)) #define inside(b, first, last) ((b) >= (first) && (b) < (last)) static int verify_group_input(struct super_block *sb, struct ext4_new_group_data *input) { struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_super_block *es = sbi->s_es; ext4_fsblk_t start = ext4_blocks_count(es); ext4_fsblk_t end = start + input->blocks_count; ext4_group_t group = input->group; ext4_fsblk_t itend = input->inode_table + sbi->s_itb_per_group; unsigned overhead; ext4_fsblk_t metaend; struct buffer_head *bh = NULL; ext4_grpblk_t free_blocks_count, offset; int err = -EINVAL; if (group != sbi->s_groups_count) { ext4_warning(sb, "Cannot add at group %u (only %u groups)", input->group, sbi->s_groups_count); return -EINVAL; } overhead = ext4_group_overhead_blocks(sb, group); metaend = start + overhead; free_blocks_count = input->blocks_count - 2 - overhead - sbi->s_itb_per_group; input->free_clusters_count = EXT4_B2C(sbi, free_blocks_count); if (test_opt(sb, DEBUG)) printk(KERN_DEBUG "EXT4-fs: adding %s group %u: %u blocks " "(%d free, %u reserved)\n", ext4_bg_has_super(sb, input->group) ? "normal" : "no-super", input->group, input->blocks_count, free_blocks_count, input->reserved_blocks); ext4_get_group_no_and_offset(sb, start, NULL, &offset); if (offset != 0) ext4_warning(sb, "Last group not full"); else if (input->reserved_blocks > input->blocks_count / 5) ext4_warning(sb, "Reserved blocks too high (%u)", input->reserved_blocks); else if (free_blocks_count < 0) ext4_warning(sb, "Bad blocks count %u", input->blocks_count); else if (IS_ERR(bh = ext4_sb_bread(sb, end - 1, 0))) { err = PTR_ERR(bh); bh = NULL; ext4_warning(sb, "Cannot read last block (%llu)", end - 1); } else if (outside(input->block_bitmap, start, end)) ext4_warning(sb, "Block bitmap not in group (block %llu)", (unsigned long long)input->block_bitmap); else if (outside(input->inode_bitmap, start, end)) ext4_warning(sb, "Inode bitmap not in group (block %llu)", (unsigned long long)input->inode_bitmap); else if (outside(input->inode_table, start, end) || outside(itend - 1, start, end)) ext4_warning(sb, "Inode table not in group (blocks %llu-%llu)", (unsigned long long)input->inode_table, itend - 1); else if (input->inode_bitmap == input->block_bitmap) ext4_warning(sb, "Block bitmap same as inode bitmap (%llu)", (unsigned long long)input->block_bitmap); else if (inside(input->block_bitmap, input->inode_table, itend)) ext4_warning(sb, "Block bitmap (%llu) in inode table " "(%llu-%llu)", (unsigned long long)input->block_bitmap, (unsigned long long)input->inode_table, itend - 1); else if (inside(input->inode_bitmap, input->inode_table, itend)) ext4_warning(sb, "Inode bitmap (%llu) in inode table " "(%llu-%llu)", (unsigned long long)input->inode_bitmap, (unsigned long long)input->inode_table, itend - 1); else if (inside(input->block_bitmap, start, metaend)) ext4_warning(sb, "Block bitmap (%llu) in GDT table (%llu-%llu)", (unsigned long long)input->block_bitmap, start, metaend - 1); else if (inside(input->inode_bitmap, start, metaend)) ext4_warning(sb, "Inode bitmap (%llu) in GDT table (%llu-%llu)", (unsigned long long)input->inode_bitmap, start, metaend - 1); else if (inside(input->inode_table, start, metaend) || inside(itend - 1, start, metaend)) ext4_warning(sb, "Inode table (%llu-%llu) overlaps GDT table " "(%llu-%llu)", (unsigned long long)input->inode_table, itend - 1, start, metaend - 1); else err = 0; brelse(bh); return err; } /* * ext4_new_flex_group_data is used by 64bit-resize interface to add a flex * group each time. */ struct ext4_new_flex_group_data { struct ext4_new_group_data *groups; /* new_group_data for groups in the flex group */ __u16 *bg_flags; /* block group flags of groups in @groups */ ext4_group_t resize_bg; /* number of allocated new_group_data */ ext4_group_t count; /* number of groups in @groups */ }; /* * Avoiding memory allocation failures due to too many groups added each time. */ #define MAX_RESIZE_BG 16384 /* * alloc_flex_gd() allocates an ext4_new_flex_group_data that satisfies the * resizing from @o_group to @n_group, its size is typically @flexbg_size. * * Returns NULL on failure otherwise address of the allocated structure. */ static struct ext4_new_flex_group_data *alloc_flex_gd(unsigned int flexbg_size, ext4_group_t o_group, ext4_group_t n_group) { ext4_group_t last_group; unsigned int max_resize_bg; struct ext4_new_flex_group_data *flex_gd; flex_gd = kmalloc(sizeof(*flex_gd), GFP_NOFS); if (flex_gd == NULL) goto out3; max_resize_bg = umin(flexbg_size, MAX_RESIZE_BG); flex_gd->resize_bg = max_resize_bg; /* Avoid allocating large 'groups' array if not needed */ last_group = o_group | (flex_gd->resize_bg - 1); if (n_group <= last_group) flex_gd->resize_bg = 1 << fls(n_group - o_group); else if (n_group - last_group < flex_gd->resize_bg) flex_gd->resize_bg = 1 << max(fls(last_group - o_group), fls(n_group - last_group)); if (WARN_ON_ONCE(flex_gd->resize_bg > max_resize_bg)) flex_gd->resize_bg = max_resize_bg; flex_gd->groups = kmalloc_array(flex_gd->resize_bg, sizeof(struct ext4_new_group_data), GFP_NOFS); if (flex_gd->groups == NULL) goto out2; flex_gd->bg_flags = kmalloc_array(flex_gd->resize_bg, sizeof(__u16), GFP_NOFS); if (flex_gd->bg_flags == NULL) goto out1; return flex_gd; out1: kfree(flex_gd->groups); out2: kfree(flex_gd); out3: return NULL; } static void free_flex_gd(struct ext4_new_flex_group_data *flex_gd) { kfree(flex_gd->bg_flags); kfree(flex_gd->groups); kfree(flex_gd); } /* * ext4_alloc_group_tables() allocates block bitmaps, inode bitmaps * and inode tables for a flex group. * * This function is used by 64bit-resize. Note that this function allocates * group tables from the 1st group of groups contained by @flexgd, which may * be a partial of a flex group. * * @sb: super block of fs to which the groups belongs * * Returns 0 on a successful allocation of the metadata blocks in the * block group. */ static int ext4_alloc_group_tables(struct super_block *sb, struct ext4_new_flex_group_data *flex_gd, unsigned int flexbg_size) { struct ext4_new_group_data *group_data = flex_gd->groups; ext4_fsblk_t start_blk; ext4_fsblk_t last_blk; ext4_group_t src_group; ext4_group_t bb_index = 0; ext4_group_t ib_index = 0; ext4_group_t it_index = 0; ext4_group_t group; ext4_group_t last_group; unsigned overhead; __u16 uninit_mask = (flexbg_size > 1) ? ~EXT4_BG_BLOCK_UNINIT : ~0; int i; BUG_ON(flex_gd->count == 0 || group_data == NULL); src_group = group_data[0].group; last_group = src_group + flex_gd->count - 1; BUG_ON((flexbg_size > 1) && ((src_group & ~(flexbg_size - 1)) != (last_group & ~(flexbg_size - 1)))); next_group: group = group_data[0].group; if (src_group >= group_data[0].group + flex_gd->count) return -ENOSPC; start_blk = ext4_group_first_block_no(sb, src_group); last_blk = start_blk + group_data[src_group - group].blocks_count; overhead = ext4_group_overhead_blocks(sb, src_group); start_blk += overhead; /* We collect contiguous blocks as much as possible. */ src_group++; for (; src_group <= last_group; src_group++) { overhead = ext4_group_overhead_blocks(sb, src_group); if (overhead == 0) last_blk += group_data[src_group - group].blocks_count; else break; } /* Allocate block bitmaps */ for (; bb_index < flex_gd->count; bb_index++) { if (start_blk >= last_blk) goto next_group; group_data[bb_index].block_bitmap = start_blk++; group = ext4_get_group_number(sb, start_blk - 1); group -= group_data[0].group; group_data[group].mdata_blocks++; flex_gd->bg_flags[group] &= uninit_mask; } /* Allocate inode bitmaps */ for (; ib_index < flex_gd->count; ib_index++) { if (start_blk >= last_blk) goto next_group; group_data[ib_index].inode_bitmap = start_blk++; group = ext4_get_group_number(sb, start_blk - 1); group -= group_data[0].group; group_data[group].mdata_blocks++; flex_gd->bg_flags[group] &= uninit_mask; } /* Allocate inode tables */ for (; it_index < flex_gd->count; it_index++) { unsigned int itb = EXT4_SB(sb)->s_itb_per_group; ext4_fsblk_t next_group_start; if (start_blk + itb > last_blk) goto next_group; group_data[it_index].inode_table = start_blk; group = ext4_get_group_number(sb, start_blk); next_group_start = ext4_group_first_block_no(sb, group + 1); group -= group_data[0].group; if (start_blk + itb > next_group_start) { flex_gd->bg_flags[group + 1] &= uninit_mask; overhead = start_blk + itb - next_group_start; group_data[group + 1].mdata_blocks += overhead; itb -= overhead; } group_data[group].mdata_blocks += itb; flex_gd->bg_flags[group] &= uninit_mask; start_blk += EXT4_SB(sb)->s_itb_per_group; } /* Update free clusters count to exclude metadata blocks */ for (i = 0; i < flex_gd->count; i++) { group_data[i].free_clusters_count -= EXT4_NUM_B2C(EXT4_SB(sb), group_data[i].mdata_blocks); } if (test_opt(sb, DEBUG)) { int i; group = group_data[0].group; printk(KERN_DEBUG "EXT4-fs: adding a flex group with " "%u groups, flexbg size is %u:\n", flex_gd->count, flexbg_size); for (i = 0; i < flex_gd->count; i++) { ext4_debug( "adding %s group %u: %u blocks (%u free, %u mdata blocks)\n", ext4_bg_has_super(sb, group + i) ? "normal" : "no-super", group + i, group_data[i].blocks_count, group_data[i].free_clusters_count, group_data[i].mdata_blocks); } } return 0; } static struct buffer_head *bclean(handle_t *handle, struct super_block *sb, ext4_fsblk_t blk) { struct buffer_head *bh; int err; bh = sb_getblk(sb, blk); if (unlikely(!bh)) return ERR_PTR(-ENOMEM); BUFFER_TRACE(bh, "get_write_access"); err = ext4_journal_get_write_access(handle, sb, bh, EXT4_JTR_NONE); if (err) { brelse(bh); bh = ERR_PTR(err); } else { memset(bh->b_data, 0, sb->s_blocksize); set_buffer_uptodate(bh); } return bh; } static int ext4_resize_ensure_credits_batch(handle_t *handle, int credits) { return ext4_journal_ensure_credits_fn(handle, credits, EXT4_MAX_TRANS_DATA, 0, 0); } /* * set_flexbg_block_bitmap() mark clusters [@first_cluster, @last_cluster] used. * * Helper function for ext4_setup_new_group_blocks() which set . * * @sb: super block * @handle: journal handle * @flex_gd: flex group data */ static int set_flexbg_block_bitmap(struct super_block *sb, handle_t *handle, struct ext4_new_flex_group_data *flex_gd, ext4_fsblk_t first_cluster, ext4_fsblk_t last_cluster) { struct ext4_sb_info *sbi = EXT4_SB(sb); ext4_group_t count = last_cluster - first_cluster + 1; ext4_group_t count2; ext4_debug("mark clusters [%llu-%llu] used\n", first_cluster, last_cluster); for (; count > 0; count -= count2, first_cluster += count2) { ext4_fsblk_t start; struct buffer_head *bh; ext4_group_t group; int err; group = ext4_get_group_number(sb, EXT4_C2B(sbi, first_cluster)); start = EXT4_B2C(sbi, ext4_group_first_block_no(sb, group)); group -= flex_gd->groups[0].group; count2 = EXT4_CLUSTERS_PER_GROUP(sb) - (first_cluster - start); if (count2 > count) count2 = count; if (flex_gd->bg_flags[group] & EXT4_BG_BLOCK_UNINIT) { BUG_ON(flex_gd->count > 1); continue; } err = ext4_resize_ensure_credits_batch(handle, 1); if (err < 0) return err; bh = sb_getblk(sb, flex_gd->groups[group].block_bitmap); if (unlikely(!bh)) return -ENOMEM; BUFFER_TRACE(bh, "get_write_access"); err = ext4_journal_get_write_access(handle, sb, bh, EXT4_JTR_NONE); if (err) { brelse(bh); return err; } ext4_debug("mark block bitmap %#04llx (+%llu/%u)\n", first_cluster, first_cluster - start, count2); mb_set_bits(bh->b_data, first_cluster - start, count2); err = ext4_handle_dirty_metadata(handle, NULL, bh); brelse(bh); if (unlikely(err)) return err; } return 0; } /* * Set up the block and inode bitmaps, and the inode table for the new groups. * This doesn't need to be part of the main transaction, since we are only * changing blocks outside the actual filesystem. We still do journaling to * ensure the recovery is correct in case of a failure just after resize. * If any part of this fails, we simply abort the resize. * * setup_new_flex_group_blocks handles a flex group as follow: * 1. copy super block and GDT, and initialize group tables if necessary. * In this step, we only set bits in blocks bitmaps for blocks taken by * super block and GDT. * 2. allocate group tables in block bitmaps, that is, set bits in block * bitmap for blocks taken by group tables. */ static int setup_new_flex_group_blocks(struct super_block *sb, struct ext4_new_flex_group_data *flex_gd) { int group_table_count[] = {1, 1, EXT4_SB(sb)->s_itb_per_group}; ext4_fsblk_t start; ext4_fsblk_t block; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_super_block *es = sbi->s_es; struct ext4_new_group_data *group_data = flex_gd->groups; __u16 *bg_flags = flex_gd->bg_flags; handle_t *handle; ext4_group_t group, count; struct buffer_head *bh = NULL; int reserved_gdb, i, j, err = 0, err2; int meta_bg; BUG_ON(!flex_gd->count || !group_data || group_data[0].group != sbi->s_groups_count); reserved_gdb = le16_to_cpu(es->s_reserved_gdt_blocks); meta_bg = ext4_has_feature_meta_bg(sb); /* This transaction may be extended/restarted along the way */ handle = ext4_journal_start_sb(sb, EXT4_HT_RESIZE, EXT4_MAX_TRANS_DATA); if (IS_ERR(handle)) return PTR_ERR(handle); group = group_data[0].group; for (i = 0; i < flex_gd->count; i++, group++) { unsigned long gdblocks; ext4_grpblk_t overhead; gdblocks = ext4_bg_num_gdb(sb, group); start = ext4_group_first_block_no(sb, group); if (meta_bg == 0 && !ext4_bg_has_super(sb, group)) goto handle_itb; if (meta_bg == 1) goto handle_itb; block = start + ext4_bg_has_super(sb, group); /* Copy all of the GDT blocks into the backup in this group */ for (j = 0; j < gdblocks; j++, block++) { struct buffer_head *gdb; ext4_debug("update backup group %#04llx\n", block); err = ext4_resize_ensure_credits_batch(handle, 1); if (err < 0) goto out; gdb = sb_getblk(sb, block); if (unlikely(!gdb)) { err = -ENOMEM; goto out; } BUFFER_TRACE(gdb, "get_write_access"); err = ext4_journal_get_write_access(handle, sb, gdb, EXT4_JTR_NONE); if (err) { brelse(gdb); goto out; } memcpy(gdb->b_data, sbi_array_rcu_deref(sbi, s_group_desc, j)->b_data, gdb->b_size); set_buffer_uptodate(gdb); err = ext4_handle_dirty_metadata(handle, NULL, gdb); if (unlikely(err)) { brelse(gdb); goto out; } brelse(gdb); } /* Zero out all of the reserved backup group descriptor * table blocks */ if (ext4_bg_has_super(sb, group)) { err = sb_issue_zeroout(sb, gdblocks + start + 1, reserved_gdb, GFP_NOFS); if (err) goto out; } handle_itb: /* Initialize group tables of the group @group */ if (!(bg_flags[i] & EXT4_BG_INODE_ZEROED)) goto handle_bb; /* Zero out all of the inode table blocks */ block = group_data[i].inode_table; ext4_debug("clear inode table blocks %#04llx -> %#04lx\n", block, sbi->s_itb_per_group); err = sb_issue_zeroout(sb, block, sbi->s_itb_per_group, GFP_NOFS); if (err) goto out; handle_bb: if (bg_flags[i] & EXT4_BG_BLOCK_UNINIT) goto handle_ib; /* Initialize block bitmap of the @group */ block = group_data[i].block_bitmap; err = ext4_resize_ensure_credits_batch(handle, 1); if (err < 0) goto out; bh = bclean(handle, sb, block); if (IS_ERR(bh)) { err = PTR_ERR(bh); goto out; } overhead = ext4_group_overhead_blocks(sb, group); if (overhead != 0) { ext4_debug("mark backup superblock %#04llx (+0)\n", start); mb_set_bits(bh->b_data, 0, EXT4_NUM_B2C(sbi, overhead)); } ext4_mark_bitmap_end(EXT4_B2C(sbi, group_data[i].blocks_count), sb->s_blocksize * 8, bh->b_data); err = ext4_handle_dirty_metadata(handle, NULL, bh); brelse(bh); if (err) goto out; handle_ib: if (bg_flags[i] & EXT4_BG_INODE_UNINIT) continue; /* Initialize inode bitmap of the @group */ block = group_data[i].inode_bitmap; err = ext4_resize_ensure_credits_batch(handle, 1); if (err < 0) goto out; /* Mark unused entries in inode bitmap used */ bh = bclean(handle, sb, block); if (IS_ERR(bh)) { err = PTR_ERR(bh); goto out; } ext4_mark_bitmap_end(EXT4_INODES_PER_GROUP(sb), sb->s_blocksize * 8, bh->b_data); err = ext4_handle_dirty_metadata(handle, NULL, bh); brelse(bh); if (err) goto out; } /* Mark group tables in block bitmap */ for (j = 0; j < GROUP_TABLE_COUNT; j++) { count = group_table_count[j]; start = (&group_data[0].block_bitmap)[j]; block = start; for (i = 1; i < flex_gd->count; i++) { block += group_table_count[j]; if (block == (&group_data[i].block_bitmap)[j]) { count += group_table_count[j]; continue; } err = set_flexbg_block_bitmap(sb, handle, flex_gd, EXT4_B2C(sbi, start), EXT4_B2C(sbi, start + count - 1)); if (err) goto out; count = group_table_count[j]; start = (&group_data[i].block_bitmap)[j]; block = start; } err = set_flexbg_block_bitmap(sb, handle, flex_gd, EXT4_B2C(sbi, start), EXT4_B2C(sbi, start + count - 1)); if (err) goto out; } out: err2 = ext4_journal_stop(handle); if (err2 && !err) err = err2; return err; } /* * Iterate through the groups which hold BACKUP superblock/GDT copies in an * ext4 filesystem. The counters should be initialized to 1, 5, and 7 before * calling this for the first time. In a sparse filesystem it will be the * sequence of powers of 3, 5, and 7: 1, 3, 5, 7, 9, 25, 27, 49, 81, ... * For a non-sparse filesystem it will be every group: 1, 2, 3, 4, ... */ unsigned int ext4_list_backups(struct super_block *sb, unsigned int *three, unsigned int *five, unsigned int *seven) { struct ext4_super_block *es = EXT4_SB(sb)->s_es; unsigned int *min = three; int mult = 3; unsigned int ret; if (ext4_has_feature_sparse_super2(sb)) { do { if (*min > 2) return UINT_MAX; ret = le32_to_cpu(es->s_backup_bgs[*min - 1]); *min += 1; } while (!ret); return ret; } if (!ext4_has_feature_sparse_super(sb)) { ret = *min; *min += 1; return ret; } if (*five < *min) { min = five; mult = 5; } if (*seven < *min) { min = seven; mult = 7; } ret = *min; *min *= mult; return ret; } /* * Check that all of the backup GDT blocks are held in the primary GDT block. * It is assumed that they are stored in group order. Returns the number of * groups in current filesystem that have BACKUPS, or -ve error code. */ static int verify_reserved_gdb(struct super_block *sb, ext4_group_t end, struct buffer_head *primary) { const ext4_fsblk_t blk = primary->b_blocknr; unsigned three = 1; unsigned five = 5; unsigned seven = 7; unsigned grp; __le32 *p = (__le32 *)primary->b_data; int gdbackups = 0; while ((grp = ext4_list_backups(sb, &three, &five, &seven)) < end) { if (le32_to_cpu(*p++) != grp * EXT4_BLOCKS_PER_GROUP(sb) + blk){ ext4_warning(sb, "reserved GDT %llu" " missing grp %d (%llu)", blk, grp, grp * (ext4_fsblk_t)EXT4_BLOCKS_PER_GROUP(sb) + blk); return -EINVAL; } if (++gdbackups > EXT4_ADDR_PER_BLOCK(sb)) return -EFBIG; } return gdbackups; } /* * Called when we need to bring a reserved group descriptor table block into * use from the resize inode. The primary copy of the new GDT block currently * is an indirect block (under the double indirect block in the resize inode). * The new backup GDT blocks will be stored as leaf blocks in this indirect * block, in group order. Even though we know all the block numbers we need, * we check to ensure that the resize inode has actually reserved these blocks. * * Don't need to update the block bitmaps because the blocks are still in use. * * We get all of the error cases out of the way, so that we are sure to not * fail once we start modifying the data on disk, because JBD has no rollback. */ static int add_new_gdb(handle_t *handle, struct inode *inode, ext4_group_t group) { struct super_block *sb = inode->i_sb; struct ext4_super_block *es = EXT4_SB(sb)->s_es; unsigned long gdb_num = group / EXT4_DESC_PER_BLOCK(sb); ext4_fsblk_t gdblock = EXT4_SB(sb)->s_sbh->b_blocknr + 1 + gdb_num; struct buffer_head **o_group_desc, **n_group_desc = NULL; struct buffer_head *dind = NULL; struct buffer_head *gdb_bh = NULL; int gdbackups; struct ext4_iloc iloc = { .bh = NULL }; __le32 *data; int err; if (test_opt(sb, DEBUG)) printk(KERN_DEBUG "EXT4-fs: ext4_add_new_gdb: adding group block %lu\n", gdb_num); gdb_bh = ext4_sb_bread(sb, gdblock, 0); if (IS_ERR(gdb_bh)) return PTR_ERR(gdb_bh); gdbackups = verify_reserved_gdb(sb, group, gdb_bh); if (gdbackups < 0) { err = gdbackups; goto errout; } data = EXT4_I(inode)->i_data + EXT4_DIND_BLOCK; dind = ext4_sb_bread(sb, le32_to_cpu(*data), 0); if (IS_ERR(dind)) { err = PTR_ERR(dind); dind = NULL; goto errout; } data = (__le32 *)dind->b_data; if (le32_to_cpu(data[gdb_num % EXT4_ADDR_PER_BLOCK(sb)]) != gdblock) { ext4_warning(sb, "new group %u GDT block %llu not reserved", group, gdblock); err = -EINVAL; goto errout; } BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get_write_access"); err = ext4_journal_get_write_access(handle, sb, EXT4_SB(sb)->s_sbh, EXT4_JTR_NONE); if (unlikely(err)) goto errout; BUFFER_TRACE(gdb_bh, "get_write_access"); err = ext4_journal_get_write_access(handle, sb, gdb_bh, EXT4_JTR_NONE); if (unlikely(err)) goto errout; BUFFER_TRACE(dind, "get_write_access"); err = ext4_journal_get_write_access(handle, sb, dind, EXT4_JTR_NONE); if (unlikely(err)) { ext4_std_error(sb, err); goto errout; } /* ext4_reserve_inode_write() gets a reference on the iloc */ err = ext4_reserve_inode_write(handle, inode, &iloc); if (unlikely(err)) goto errout; n_group_desc = kvmalloc((gdb_num + 1) * sizeof(struct buffer_head *), GFP_KERNEL); if (!n_group_desc) { err = -ENOMEM; ext4_warning(sb, "not enough memory for %lu groups", gdb_num + 1); goto errout; } /* * Finally, we have all of the possible failures behind us... * * Remove new GDT block from inode double-indirect block and clear out * the new GDT block for use (which also "frees" the backup GDT blocks * from the reserved inode). We don't need to change the bitmaps for * these blocks, because they are marked as in-use from being in the * reserved inode, and will become GDT blocks (primary and backup). */ data[gdb_num % EXT4_ADDR_PER_BLOCK(sb)] = 0; err = ext4_handle_dirty_metadata(handle, NULL, dind); if (unlikely(err)) { ext4_std_error(sb, err); goto errout; } inode->i_blocks -= (gdbackups + 1) * sb->s_blocksize >> (9 - EXT4_SB(sb)->s_cluster_bits); ext4_mark_iloc_dirty(handle, inode, &iloc); memset(gdb_bh->b_data, 0, sb->s_blocksize); err = ext4_handle_dirty_metadata(handle, NULL, gdb_bh); if (unlikely(err)) { ext4_std_error(sb, err); iloc.bh = NULL; goto errout; } brelse(dind); rcu_read_lock(); o_group_desc = rcu_dereference(EXT4_SB(sb)->s_group_desc); memcpy(n_group_desc, o_group_desc, EXT4_SB(sb)->s_gdb_count * sizeof(struct buffer_head *)); rcu_read_unlock(); n_group_desc[gdb_num] = gdb_bh; rcu_assign_pointer(EXT4_SB(sb)->s_group_desc, n_group_desc); EXT4_SB(sb)->s_gdb_count++; ext4_kvfree_array_rcu(o_group_desc); lock_buffer(EXT4_SB(sb)->s_sbh); le16_add_cpu(&es->s_reserved_gdt_blocks, -1); ext4_superblock_csum_set(sb); unlock_buffer(EXT4_SB(sb)->s_sbh); err = ext4_handle_dirty_metadata(handle, NULL, EXT4_SB(sb)->s_sbh); if (err) ext4_std_error(sb, err); return err; errout: kvfree(n_group_desc); brelse(iloc.bh); brelse(dind); brelse(gdb_bh); ext4_debug("leaving with error %d\n", err); return err; } /* * If there is no available space in the existing block group descriptors for * the new block group and there are no reserved block group descriptors, then * the meta_bg feature will get enabled, and es->s_first_meta_bg will get set * to the first block group that is managed using meta_bg and s_first_meta_bg * must be a multiple of EXT4_DESC_PER_BLOCK(sb). * This function will be called when first group of meta_bg is added to bring * new group descriptors block of new added meta_bg. */ static int add_new_gdb_meta_bg(struct super_block *sb, handle_t *handle, ext4_group_t group) { ext4_fsblk_t gdblock; struct buffer_head *gdb_bh; struct buffer_head **o_group_desc, **n_group_desc; unsigned long gdb_num = group / EXT4_DESC_PER_BLOCK(sb); int err; gdblock = ext4_group_first_block_no(sb, group) + ext4_bg_has_super(sb, group); gdb_bh = ext4_sb_bread(sb, gdblock, 0); if (IS_ERR(gdb_bh)) return PTR_ERR(gdb_bh); n_group_desc = kvmalloc((gdb_num + 1) * sizeof(struct buffer_head *), GFP_KERNEL); if (!n_group_desc) { brelse(gdb_bh); err = -ENOMEM; ext4_warning(sb, "not enough memory for %lu groups", gdb_num + 1); return err; } rcu_read_lock(); o_group_desc = rcu_dereference(EXT4_SB(sb)->s_group_desc); memcpy(n_group_desc, o_group_desc, EXT4_SB(sb)->s_gdb_count * sizeof(struct buffer_head *)); rcu_read_unlock(); n_group_desc[gdb_num] = gdb_bh; BUFFER_TRACE(gdb_bh, "get_write_access"); err = ext4_journal_get_write_access(handle, sb, gdb_bh, EXT4_JTR_NONE); if (err) { kvfree(n_group_desc); brelse(gdb_bh); return err; } rcu_assign_pointer(EXT4_SB(sb)->s_group_desc, n_group_desc); EXT4_SB(sb)->s_gdb_count++; ext4_kvfree_array_rcu(o_group_desc); return err; } /* * Called when we are adding a new group which has a backup copy of each of * the GDT blocks (i.e. sparse group) and there are reserved GDT blocks. * We need to add these reserved backup GDT blocks to the resize inode, so * that they are kept for future resizing and not allocated to files. * * Each reserved backup GDT block will go into a different indirect block. * The indirect blocks are actually the primary reserved GDT blocks, * so we know in advance what their block numbers are. We only get the * double-indirect block to verify it is pointing to the primary reserved * GDT blocks so we don't overwrite a data block by accident. The reserved * backup GDT blocks are stored in their reserved primary GDT block. */ static int reserve_backup_gdb(handle_t *handle, struct inode *inode, ext4_group_t group) { struct super_block *sb = inode->i_sb; int reserved_gdb =le16_to_cpu(EXT4_SB(sb)->s_es->s_reserved_gdt_blocks); int cluster_bits = EXT4_SB(sb)->s_cluster_bits; struct buffer_head **primary; struct buffer_head *dind; struct ext4_iloc iloc; ext4_fsblk_t blk; __le32 *data, *end; int gdbackups = 0; int res, i; int err; primary = kmalloc_array(reserved_gdb, sizeof(*primary), GFP_NOFS); if (!primary) return -ENOMEM; data = EXT4_I(inode)->i_data + EXT4_DIND_BLOCK; dind = ext4_sb_bread(sb, le32_to_cpu(*data), 0); if (IS_ERR(dind)) { err = PTR_ERR(dind); dind = NULL; goto exit_free; } blk = EXT4_SB(sb)->s_sbh->b_blocknr + 1 + EXT4_SB(sb)->s_gdb_count; data = (__le32 *)dind->b_data + (EXT4_SB(sb)->s_gdb_count % EXT4_ADDR_PER_BLOCK(sb)); end = (__le32 *)dind->b_data + EXT4_ADDR_PER_BLOCK(sb); /* Get each reserved primary GDT block and verify it holds backups */ for (res = 0; res < reserved_gdb; res++, blk++) { if (le32_to_cpu(*data) != blk) { ext4_warning(sb, "reserved block %llu" " not at offset %ld", blk, (long)(data - (__le32 *)dind->b_data)); err = -EINVAL; goto exit_bh; } primary[res] = ext4_sb_bread(sb, blk, 0); if (IS_ERR(primary[res])) { err = PTR_ERR(primary[res]); primary[res] = NULL; goto exit_bh; } gdbackups = verify_reserved_gdb(sb, group, primary[res]); if (gdbackups < 0) { brelse(primary[res]); err = gdbackups; goto exit_bh; } if (++data >= end) data = (__le32 *)dind->b_data; } for (i = 0; i < reserved_gdb; i++) { BUFFER_TRACE(primary[i], "get_write_access"); if ((err = ext4_journal_get_write_access(handle, sb, primary[i], EXT4_JTR_NONE))) goto exit_bh; } if ((err = ext4_reserve_inode_write(handle, inode, &iloc))) goto exit_bh; /* * Finally we can add each of the reserved backup GDT blocks from * the new group to its reserved primary GDT block. */ blk = group * EXT4_BLOCKS_PER_GROUP(sb); for (i = 0; i < reserved_gdb; i++) { int err2; data = (__le32 *)primary[i]->b_data; data[gdbackups] = cpu_to_le32(blk + primary[i]->b_blocknr); err2 = ext4_handle_dirty_metadata(handle, NULL, primary[i]); if (!err) err = err2; } inode->i_blocks += reserved_gdb * sb->s_blocksize >> (9 - cluster_bits); ext4_mark_iloc_dirty(handle, inode, &iloc); exit_bh: while (--res >= 0) brelse(primary[res]); brelse(dind); exit_free: kfree(primary); return err; } static inline void ext4_set_block_group_nr(struct super_block *sb, char *data, ext4_group_t group) { struct ext4_super_block *es = (struct ext4_super_block *) data; es->s_block_group_nr = cpu_to_le16(group); if (ext4_has_metadata_csum(sb)) es->s_checksum = ext4_superblock_csum(sb, es); } /* * Update the backup copies of the ext4 metadata. These don't need to be part * of the main resize transaction, because e2fsck will re-write them if there * is a problem (basically only OOM will cause a problem). However, we * _should_ update the backups if possible, in case the primary gets trashed * for some reason and we need to run e2fsck from a backup superblock. The * important part is that the new block and inode counts are in the backup * superblocks, and the location of the new group metadata in the GDT backups. * * We do not need take the s_resize_lock for this, because these * blocks are not otherwise touched by the filesystem code when it is * mounted. We don't need to worry about last changing from * sbi->s_groups_count, because the worst that can happen is that we * do not copy the full number of backups at this time. The resize * which changed s_groups_count will backup again. */ static void update_backups(struct super_block *sb, sector_t blk_off, char *data, int size, int meta_bg) { struct ext4_sb_info *sbi = EXT4_SB(sb); ext4_group_t last; const int bpg = EXT4_BLOCKS_PER_GROUP(sb); unsigned three = 1; unsigned five = 5; unsigned seven = 7; ext4_group_t group = 0; int rest = sb->s_blocksize - size; handle_t *handle; int err = 0, err2; handle = ext4_journal_start_sb(sb, EXT4_HT_RESIZE, EXT4_MAX_TRANS_DATA); if (IS_ERR(handle)) { group = 1; err = PTR_ERR(handle); goto exit_err; } if (meta_bg == 0) { group = ext4_list_backups(sb, &three, &five, &seven); last = sbi->s_groups_count; } else { group = ext4_get_group_number(sb, blk_off) + 1; last = (ext4_group_t)(group + EXT4_DESC_PER_BLOCK(sb) - 2); } while (group < sbi->s_groups_count) { struct buffer_head *bh; ext4_fsblk_t backup_block; int has_super = ext4_bg_has_super(sb, group); ext4_fsblk_t first_block = ext4_group_first_block_no(sb, group); /* Out of journal space, and can't get more - abort - so sad */ err = ext4_resize_ensure_credits_batch(handle, 1); if (err < 0) break; if (meta_bg == 0) backup_block = ((ext4_fsblk_t)group) * bpg + blk_off; else backup_block = first_block + has_super; bh = sb_getblk(sb, backup_block); if (unlikely(!bh)) { err = -ENOMEM; break; } ext4_debug("update metadata backup %llu(+%llu)\n", backup_block, backup_block - ext4_group_first_block_no(sb, group)); BUFFER_TRACE(bh, "get_write_access"); if ((err = ext4_journal_get_write_access(handle, sb, bh, EXT4_JTR_NONE))) { brelse(bh); break; } lock_buffer(bh); memcpy(bh->b_data, data, size); if (rest) memset(bh->b_data + size, 0, rest); if (has_super && (backup_block == first_block)) ext4_set_block_group_nr(sb, bh->b_data, group); set_buffer_uptodate(bh); unlock_buffer(bh); err = ext4_handle_dirty_metadata(handle, NULL, bh); if (unlikely(err)) ext4_std_error(sb, err); brelse(bh); if (meta_bg == 0) group = ext4_list_backups(sb, &three, &five, &seven); else if (group == last) break; else group = last; } if ((err2 = ext4_journal_stop(handle)) && !err) err = err2; /* * Ugh! Need to have e2fsck write the backup copies. It is too * late to revert the resize, we shouldn't fail just because of * the backup copies (they are only needed in case of corruption). * * However, if we got here we have a journal problem too, so we * can't really start a transaction to mark the superblock. * Chicken out and just set the flag on the hope it will be written * to disk, and if not - we will simply wait until next fsck. */ exit_err: if (err) { ext4_warning(sb, "can't update backup for group %u (err %d), " "forcing fsck on next reboot", group, err); sbi->s_mount_state &= ~EXT4_VALID_FS; sbi->s_es->s_state &= cpu_to_le16(~EXT4_VALID_FS); mark_buffer_dirty(sbi->s_sbh); } } /* * ext4_add_new_descs() adds @count group descriptor of groups * starting at @group * * @handle: journal handle * @sb: super block * @group: the group no. of the first group desc to be added * @resize_inode: the resize inode * @count: number of group descriptors to be added */ static int ext4_add_new_descs(handle_t *handle, struct super_block *sb, ext4_group_t group, struct inode *resize_inode, ext4_group_t count) { struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_super_block *es = sbi->s_es; struct buffer_head *gdb_bh; int i, gdb_off, gdb_num, err = 0; int meta_bg; meta_bg = ext4_has_feature_meta_bg(sb); for (i = 0; i < count; i++, group++) { int reserved_gdb = ext4_bg_has_super(sb, group) ? le16_to_cpu(es->s_reserved_gdt_blocks) : 0; gdb_off = group % EXT4_DESC_PER_BLOCK(sb); gdb_num = group / EXT4_DESC_PER_BLOCK(sb); /* * We will only either add reserved group blocks to a backup group * or remove reserved blocks for the first group in a new group block. * Doing both would be mean more complex code, and sane people don't * use non-sparse filesystems anymore. This is already checked above. */ if (gdb_off) { gdb_bh = sbi_array_rcu_deref(sbi, s_group_desc, gdb_num); BUFFER_TRACE(gdb_bh, "get_write_access"); err = ext4_journal_get_write_access(handle, sb, gdb_bh, EXT4_JTR_NONE); if (!err && reserved_gdb && ext4_bg_num_gdb(sb, group)) err = reserve_backup_gdb(handle, resize_inode, group); } else if (meta_bg != 0) { err = add_new_gdb_meta_bg(sb, handle, group); } else { err = add_new_gdb(handle, resize_inode, group); } if (err) break; } return err; } static struct buffer_head *ext4_get_bitmap(struct super_block *sb, __u64 block) { struct buffer_head *bh = sb_getblk(sb, block); if (unlikely(!bh)) return NULL; if (!bh_uptodate_or_lock(bh)) { if (ext4_read_bh(bh, 0, NULL, false) < 0) { brelse(bh); return NULL; } } return bh; } static int ext4_set_bitmap_checksums(struct super_block *sb, struct ext4_group_desc *gdp, struct ext4_new_group_data *group_data) { struct buffer_head *bh; if (!ext4_has_metadata_csum(sb)) return 0; bh = ext4_get_bitmap(sb, group_data->inode_bitmap); if (!bh) return -EIO; ext4_inode_bitmap_csum_set(sb, gdp, bh); brelse(bh); bh = ext4_get_bitmap(sb, group_data->block_bitmap); if (!bh) return -EIO; ext4_block_bitmap_csum_set(sb, gdp, bh); brelse(bh); return 0; } /* * ext4_setup_new_descs() will set up the group descriptor descriptors of a flex bg */ static int ext4_setup_new_descs(handle_t *handle, struct super_block *sb, struct ext4_new_flex_group_data *flex_gd) { struct ext4_new_group_data *group_data = flex_gd->groups; struct ext4_group_desc *gdp; struct ext4_sb_info *sbi = EXT4_SB(sb); struct buffer_head *gdb_bh; ext4_group_t group; __u16 *bg_flags = flex_gd->bg_flags; int i, gdb_off, gdb_num, err = 0; for (i = 0; i < flex_gd->count; i++, group_data++, bg_flags++) { group = group_data->group; gdb_off = group % EXT4_DESC_PER_BLOCK(sb); gdb_num = group / EXT4_DESC_PER_BLOCK(sb); /* * get_write_access() has been called on gdb_bh by ext4_add_new_desc(). */ gdb_bh = sbi_array_rcu_deref(sbi, s_group_desc, gdb_num); /* Update group descriptor block for new group */ gdp = (struct ext4_group_desc *)(gdb_bh->b_data + gdb_off * EXT4_DESC_SIZE(sb)); memset(gdp, 0, EXT4_DESC_SIZE(sb)); ext4_block_bitmap_set(sb, gdp, group_data->block_bitmap); ext4_inode_bitmap_set(sb, gdp, group_data->inode_bitmap); err = ext4_set_bitmap_checksums(sb, gdp, group_data); if (err) { ext4_std_error(sb, err); break; } ext4_inode_table_set(sb, gdp, group_data->inode_table); ext4_free_group_clusters_set(sb, gdp, group_data->free_clusters_count); ext4_free_inodes_set(sb, gdp, EXT4_INODES_PER_GROUP(sb)); if (ext4_has_group_desc_csum(sb)) ext4_itable_unused_set(sb, gdp, EXT4_INODES_PER_GROUP(sb)); gdp->bg_flags = cpu_to_le16(*bg_flags); ext4_group_desc_csum_set(sb, group, gdp); err = ext4_handle_dirty_metadata(handle, NULL, gdb_bh); if (unlikely(err)) { ext4_std_error(sb, err); break; } /* * We can allocate memory for mb_alloc based on the new group * descriptor */ err = ext4_mb_add_groupinfo(sb, group, gdp); if (err) break; } return err; } static void ext4_add_overhead(struct super_block *sb, const ext4_fsblk_t overhead) { struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_super_block *es = sbi->s_es; sbi->s_overhead += overhead; es->s_overhead_clusters = cpu_to_le32(sbi->s_overhead); smp_wmb(); } /* * ext4_update_super() updates the super block so that the newly added * groups can be seen by the filesystem. * * @sb: super block * @flex_gd: new added groups */ static void ext4_update_super(struct super_block *sb, struct ext4_new_flex_group_data *flex_gd) { ext4_fsblk_t blocks_count = 0; ext4_fsblk_t free_blocks = 0; ext4_fsblk_t reserved_blocks = 0; struct ext4_new_group_data *group_data = flex_gd->groups; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_super_block *es = sbi->s_es; int i; BUG_ON(flex_gd->count == 0 || group_data == NULL); /* * Make the new blocks and inodes valid next. We do this before * increasing the group count so that once the group is enabled, * all of its blocks and inodes are already valid. * * We always allocate group-by-group, then block-by-block or * inode-by-inode within a group, so enabling these * blocks/inodes before the group is live won't actually let us * allocate the new space yet. */ for (i = 0; i < flex_gd->count; i++) { blocks_count += group_data[i].blocks_count; free_blocks += EXT4_C2B(sbi, group_data[i].free_clusters_count); } reserved_blocks = ext4_r_blocks_count(es) * 100; reserved_blocks = div64_u64(reserved_blocks, ext4_blocks_count(es)); reserved_blocks *= blocks_count; do_div(reserved_blocks, 100); lock_buffer(sbi->s_sbh); ext4_blocks_count_set(es, ext4_blocks_count(es) + blocks_count); ext4_free_blocks_count_set(es, ext4_free_blocks_count(es) + free_blocks); le32_add_cpu(&es->s_inodes_count, EXT4_INODES_PER_GROUP(sb) * flex_gd->count); le32_add_cpu(&es->s_free_inodes_count, EXT4_INODES_PER_GROUP(sb) * flex_gd->count); ext4_debug("free blocks count %llu", ext4_free_blocks_count(es)); /* * We need to protect s_groups_count against other CPUs seeing * inconsistent state in the superblock. * * The precise rules we use are: * * * Writers must perform a smp_wmb() after updating all * dependent data and before modifying the groups count * * * Readers must perform an smp_rmb() after reading the groups * count and before reading any dependent data. * * NB. These rules can be relaxed when checking the group count * while freeing data, as we can only allocate from a block * group after serialising against the group count, and we can * only then free after serialising in turn against that * allocation. */ smp_wmb(); /* Update the global fs size fields */ sbi->s_groups_count += flex_gd->count; sbi->s_blockfile_groups = min_t(ext4_group_t, sbi->s_groups_count, (EXT4_MAX_BLOCK_FILE_PHYS / EXT4_BLOCKS_PER_GROUP(sb))); /* Update the reserved block counts only once the new group is * active. */ ext4_r_blocks_count_set(es, ext4_r_blocks_count(es) + reserved_blocks); /* Update the free space counts */ percpu_counter_add(&sbi->s_freeclusters_counter, EXT4_NUM_B2C(sbi, free_blocks)); percpu_counter_add(&sbi->s_freeinodes_counter, EXT4_INODES_PER_GROUP(sb) * flex_gd->count); ext4_debug("free blocks count %llu", percpu_counter_read(&sbi->s_freeclusters_counter)); if (ext4_has_feature_flex_bg(sb) && sbi->s_log_groups_per_flex) { ext4_group_t flex_group; struct flex_groups *fg; flex_group = ext4_flex_group(sbi, group_data[0].group); fg = sbi_array_rcu_deref(sbi, s_flex_groups, flex_group); atomic64_add(EXT4_NUM_B2C(sbi, free_blocks), &fg->free_clusters); atomic_add(EXT4_INODES_PER_GROUP(sb) * flex_gd->count, &fg->free_inodes); } /* * Update the fs overhead information. * * For bigalloc, if the superblock already has a properly calculated * overhead, update it with a value based on numbers already computed * above for the newly allocated capacity. */ if (ext4_has_feature_bigalloc(sb) && (sbi->s_overhead != 0)) ext4_add_overhead(sb, EXT4_NUM_B2C(sbi, blocks_count - free_blocks)); else ext4_calculate_overhead(sb); es->s_overhead_clusters = cpu_to_le32(sbi->s_overhead); ext4_superblock_csum_set(sb); unlock_buffer(sbi->s_sbh); if (test_opt(sb, DEBUG)) printk(KERN_DEBUG "EXT4-fs: added group %u:" "%llu blocks(%llu free %llu reserved)\n", flex_gd->count, blocks_count, free_blocks, reserved_blocks); } /* Add a flex group to an fs. Ensure we handle all possible error conditions * _before_ we start modifying the filesystem, because we cannot abort the * transaction and not have it write the data to disk. */ static int ext4_flex_group_add(struct super_block *sb, struct inode *resize_inode, struct ext4_new_flex_group_data *flex_gd) { struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_super_block *es = sbi->s_es; ext4_fsblk_t o_blocks_count; ext4_grpblk_t last; ext4_group_t group; handle_t *handle; unsigned reserved_gdb; int err = 0, err2 = 0, credit; BUG_ON(!flex_gd->count || !flex_gd->groups || !flex_gd->bg_flags); reserved_gdb = le16_to_cpu(es->s_reserved_gdt_blocks); o_blocks_count = ext4_blocks_count(es); ext4_get_group_no_and_offset(sb, o_blocks_count, &group, &last); BUG_ON(last); err = setup_new_flex_group_blocks(sb, flex_gd); if (err) goto exit; /* * We will always be modifying at least the superblock and GDT * blocks. If we are adding a group past the last current GDT block, * we will also modify the inode and the dindirect block. If we * are adding a group with superblock/GDT backups we will also * modify each of the reserved GDT dindirect blocks. */ credit = 3; /* sb, resize inode, resize inode dindirect */ /* GDT blocks */ credit += 1 + DIV_ROUND_UP(flex_gd->count, EXT4_DESC_PER_BLOCK(sb)); credit += reserved_gdb; /* Reserved GDT dindirect blocks */ handle = ext4_journal_start_sb(sb, EXT4_HT_RESIZE, credit); if (IS_ERR(handle)) { err = PTR_ERR(handle); goto exit; } BUFFER_TRACE(sbi->s_sbh, "get_write_access"); err = ext4_journal_get_write_access(handle, sb, sbi->s_sbh, EXT4_JTR_NONE); if (err) goto exit_journal; group = flex_gd->groups[0].group; BUG_ON(group != sbi->s_groups_count); err = ext4_add_new_descs(handle, sb, group, resize_inode, flex_gd->count); if (err) goto exit_journal; err = ext4_setup_new_descs(handle, sb, flex_gd); if (err) goto exit_journal; ext4_update_super(sb, flex_gd); err = ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh); exit_journal: err2 = ext4_journal_stop(handle); if (!err) err = err2; if (!err) { int gdb_num = group / EXT4_DESC_PER_BLOCK(sb); int gdb_num_end = ((group + flex_gd->count - 1) / EXT4_DESC_PER_BLOCK(sb)); int meta_bg = ext4_has_feature_meta_bg(sb) && gdb_num >= le32_to_cpu(es->s_first_meta_bg); sector_t padding_blocks = meta_bg ? 0 : sbi->s_sbh->b_blocknr - ext4_group_first_block_no(sb, 0); update_backups(sb, ext4_group_first_block_no(sb, 0), (char *)es, sizeof(struct ext4_super_block), 0); for (; gdb_num <= gdb_num_end; gdb_num++) { struct buffer_head *gdb_bh; gdb_bh = sbi_array_rcu_deref(sbi, s_group_desc, gdb_num); update_backups(sb, gdb_bh->b_blocknr - padding_blocks, gdb_bh->b_data, gdb_bh->b_size, meta_bg); } } exit: return err; } static int ext4_setup_next_flex_gd(struct super_block *sb, struct ext4_new_flex_group_data *flex_gd, ext4_fsblk_t n_blocks_count) { struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_super_block *es = sbi->s_es; struct ext4_new_group_data *group_data = flex_gd->groups; ext4_fsblk_t o_blocks_count; ext4_group_t n_group; ext4_group_t group; ext4_group_t last_group; ext4_grpblk_t last; ext4_grpblk_t clusters_per_group; unsigned long i; clusters_per_group = EXT4_CLUSTERS_PER_GROUP(sb); o_blocks_count = ext4_blocks_count(es); if (o_blocks_count == n_blocks_count) return 0; ext4_get_group_no_and_offset(sb, o_blocks_count, &group, &last); BUG_ON(last); ext4_get_group_no_and_offset(sb, n_blocks_count - 1, &n_group, &last); last_group = group | (flex_gd->resize_bg - 1); if (last_group > n_group) last_group = n_group; flex_gd->count = last_group - group + 1; for (i = 0; i < flex_gd->count; i++) { int overhead; group_data[i].group = group + i; group_data[i].blocks_count = EXT4_BLOCKS_PER_GROUP(sb); overhead = ext4_group_overhead_blocks(sb, group + i); group_data[i].mdata_blocks = overhead; group_data[i].free_clusters_count = EXT4_CLUSTERS_PER_GROUP(sb); if (ext4_has_group_desc_csum(sb)) { flex_gd->bg_flags[i] = EXT4_BG_BLOCK_UNINIT | EXT4_BG_INODE_UNINIT; if (!test_opt(sb, INIT_INODE_TABLE)) flex_gd->bg_flags[i] |= EXT4_BG_INODE_ZEROED; } else flex_gd->bg_flags[i] = EXT4_BG_INODE_ZEROED; } if (last_group == n_group && ext4_has_group_desc_csum(sb)) /* We need to initialize block bitmap of last group. */ flex_gd->bg_flags[i - 1] &= ~EXT4_BG_BLOCK_UNINIT; if ((last_group == n_group) && (last != clusters_per_group - 1)) { group_data[i - 1].blocks_count = EXT4_C2B(sbi, last + 1); group_data[i - 1].free_clusters_count -= clusters_per_group - last - 1; } return 1; } /* Add group descriptor data to an existing or new group descriptor block. * Ensure we handle all possible error conditions _before_ we start modifying * the filesystem, because we cannot abort the transaction and not have it * write the data to disk. * * If we are on a GDT block boundary, we need to get the reserved GDT block. * Otherwise, we may need to add backup GDT blocks for a sparse group. * * We only need to hold the superblock lock while we are actually adding * in the new group's counts to the superblock. Prior to that we have * not really "added" the group at all. We re-check that we are still * adding in the last group in case things have changed since verifying. */ int ext4_group_add(struct super_block *sb, struct ext4_new_group_data *input) { struct ext4_new_flex_group_data flex_gd; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_super_block *es = sbi->s_es; int reserved_gdb = ext4_bg_has_super(sb, input->group) ? le16_to_cpu(es->s_reserved_gdt_blocks) : 0; struct inode *inode = NULL; int gdb_off; int err; __u16 bg_flags = 0; gdb_off = input->group % EXT4_DESC_PER_BLOCK(sb); if (gdb_off == 0 && !ext4_has_feature_sparse_super(sb)) { ext4_warning(sb, "Can't resize non-sparse filesystem further"); return -EPERM; } if (ext4_blocks_count(es) + input->blocks_count < ext4_blocks_count(es)) { ext4_warning(sb, "blocks_count overflow"); return -EINVAL; } if (le32_to_cpu(es->s_inodes_count) + EXT4_INODES_PER_GROUP(sb) < le32_to_cpu(es->s_inodes_count)) { ext4_warning(sb, "inodes_count overflow"); return -EINVAL; } if (reserved_gdb || gdb_off == 0) { if (!ext4_has_feature_resize_inode(sb) || !le16_to_cpu(es->s_reserved_gdt_blocks)) { ext4_warning(sb, "No reserved GDT blocks, can't resize"); return -EPERM; } inode = ext4_iget(sb, EXT4_RESIZE_INO, EXT4_IGET_SPECIAL); if (IS_ERR(inode)) { ext4_warning(sb, "Error opening resize inode"); return PTR_ERR(inode); } } err = verify_group_input(sb, input); if (err) goto out; err = ext4_alloc_flex_bg_array(sb, input->group + 1); if (err) goto out; err = ext4_mb_alloc_groupinfo(sb, input->group + 1); if (err) goto out; flex_gd.count = 1; flex_gd.groups = input; flex_gd.bg_flags = &bg_flags; err = ext4_flex_group_add(sb, inode, &flex_gd); out: iput(inode); return err; } /* ext4_group_add */ /* * extend a group without checking assuming that checking has been done. */ static int ext4_group_extend_no_check(struct super_block *sb, ext4_fsblk_t o_blocks_count, ext4_grpblk_t add) { struct ext4_super_block *es = EXT4_SB(sb)->s_es; handle_t *handle; int err = 0, err2; /* We will update the superblock, one block bitmap, and * one group descriptor via ext4_group_add_blocks(). */ handle = ext4_journal_start_sb(sb, EXT4_HT_RESIZE, 3); if (IS_ERR(handle)) { err = PTR_ERR(handle); ext4_warning(sb, "error %d on journal start", err); return err; } BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get_write_access"); err = ext4_journal_get_write_access(handle, sb, EXT4_SB(sb)->s_sbh, EXT4_JTR_NONE); if (err) { ext4_warning(sb, "error %d on journal write access", err); goto errout; } lock_buffer(EXT4_SB(sb)->s_sbh); ext4_blocks_count_set(es, o_blocks_count + add); ext4_free_blocks_count_set(es, ext4_free_blocks_count(es) + add); ext4_superblock_csum_set(sb); unlock_buffer(EXT4_SB(sb)->s_sbh); ext4_debug("freeing blocks %llu through %llu\n", o_blocks_count, o_blocks_count + add); /* We add the blocks to the bitmap and set the group need init bit */ err = ext4_group_add_blocks(handle, sb, o_blocks_count, add); if (err) goto errout; ext4_handle_dirty_metadata(handle, NULL, EXT4_SB(sb)->s_sbh); ext4_debug("freed blocks %llu through %llu\n", o_blocks_count, o_blocks_count + add); errout: err2 = ext4_journal_stop(handle); if (err2 && !err) err = err2; if (!err) { if (test_opt(sb, DEBUG)) printk(KERN_DEBUG "EXT4-fs: extended group to %llu " "blocks\n", ext4_blocks_count(es)); update_backups(sb, ext4_group_first_block_no(sb, 0), (char *)es, sizeof(struct ext4_super_block), 0); } return err; } /* * Extend the filesystem to the new number of blocks specified. This entry * point is only used to extend the current filesystem to the end of the last * existing group. It can be accessed via ioctl, or by "remount,resize=<size>" * for emergencies (because it has no dependencies on reserved blocks). * * If we _really_ wanted, we could use default values to call ext4_group_add() * allow the "remount" trick to work for arbitrary resizing, assuming enough * GDT blocks are reserved to grow to the desired size. */ int ext4_group_extend(struct super_block *sb, struct ext4_super_block *es, ext4_fsblk_t n_blocks_count) { ext4_fsblk_t o_blocks_count; ext4_grpblk_t last; ext4_grpblk_t add; struct buffer_head *bh; ext4_group_t group; o_blocks_count = ext4_blocks_count(es); if (test_opt(sb, DEBUG)) ext4_msg(sb, KERN_DEBUG, "extending last group from %llu to %llu blocks", o_blocks_count, n_blocks_count); if (n_blocks_count == 0 || n_blocks_count == o_blocks_count) return 0; if (n_blocks_count > (sector_t)(~0ULL) >> (sb->s_blocksize_bits - 9)) { ext4_msg(sb, KERN_ERR, "filesystem too large to resize to %llu blocks safely", n_blocks_count); return -EINVAL; } if (n_blocks_count < o_blocks_count) { ext4_warning(sb, "can't shrink FS - resize aborted"); return -EINVAL; } /* Handle the remaining blocks in the last group only. */ ext4_get_group_no_and_offset(sb, o_blocks_count, &group, &last); if (last == 0) { ext4_warning(sb, "need to use ext2online to resize further"); return -EPERM; } add = EXT4_BLOCKS_PER_GROUP(sb) - last; if (o_blocks_count + add < o_blocks_count) { ext4_warning(sb, "blocks_count overflow"); return -EINVAL; } if (o_blocks_count + add > n_blocks_count) add = n_blocks_count - o_blocks_count; if (o_blocks_count + add < n_blocks_count) ext4_warning(sb, "will only finish group (%llu blocks, %u new)", o_blocks_count + add, add); /* See if the device is actually as big as what was requested */ bh = ext4_sb_bread(sb, o_blocks_count + add - 1, 0); if (IS_ERR(bh)) { ext4_warning(sb, "can't read last block, resize aborted"); return -ENOSPC; } brelse(bh); return ext4_group_extend_no_check(sb, o_blocks_count, add); } /* ext4_group_extend */ static int num_desc_blocks(struct super_block *sb, ext4_group_t groups) { return (groups + EXT4_DESC_PER_BLOCK(sb) - 1) / EXT4_DESC_PER_BLOCK(sb); } /* * Release the resize inode and drop the resize_inode feature if there * are no more reserved gdt blocks, and then convert the file system * to enable meta_bg */ static int ext4_convert_meta_bg(struct super_block *sb, struct inode *inode) { handle_t *handle; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_super_block *es = sbi->s_es; struct ext4_inode_info *ei = EXT4_I(inode); ext4_fsblk_t nr; int i, ret, err = 0; int credits = 1; ext4_msg(sb, KERN_INFO, "Converting file system to meta_bg"); if (inode) { if (es->s_reserved_gdt_blocks) { ext4_error(sb, "Unexpected non-zero " "s_reserved_gdt_blocks"); return -EPERM; } /* Do a quick sanity check of the resize inode */ if (inode->i_blocks != 1 << (inode->i_blkbits - (9 - sbi->s_cluster_bits))) goto invalid_resize_inode; for (i = 0; i < EXT4_N_BLOCKS; i++) { if (i == EXT4_DIND_BLOCK) { if (ei->i_data[i]) continue; else goto invalid_resize_inode; } if (ei->i_data[i]) goto invalid_resize_inode; } credits += 3; /* block bitmap, bg descriptor, resize inode */ } handle = ext4_journal_start_sb(sb, EXT4_HT_RESIZE, credits); if (IS_ERR(handle)) return PTR_ERR(handle); BUFFER_TRACE(sbi->s_sbh, "get_write_access"); err = ext4_journal_get_write_access(handle, sb, sbi->s_sbh, EXT4_JTR_NONE); if (err) goto errout; lock_buffer(sbi->s_sbh); ext4_clear_feature_resize_inode(sb); ext4_set_feature_meta_bg(sb); sbi->s_es->s_first_meta_bg = cpu_to_le32(num_desc_blocks(sb, sbi->s_groups_count)); ext4_superblock_csum_set(sb); unlock_buffer(sbi->s_sbh); err = ext4_handle_dirty_metadata(handle, NULL, sbi->s_sbh); if (err) { ext4_std_error(sb, err); goto errout; } if (inode) { nr = le32_to_cpu(ei->i_data[EXT4_DIND_BLOCK]); ext4_free_blocks(handle, inode, NULL, nr, 1, EXT4_FREE_BLOCKS_METADATA | EXT4_FREE_BLOCKS_FORGET); ei->i_data[EXT4_DIND_BLOCK] = 0; inode->i_blocks = 0; err = ext4_mark_inode_dirty(handle, inode); if (err) ext4_std_error(sb, err); } errout: ret = ext4_journal_stop(handle); return err ? err : ret; invalid_resize_inode: ext4_error(sb, "corrupted/inconsistent resize inode"); return -EINVAL; } /* * ext4_resize_fs() resizes a fs to new size specified by @n_blocks_count * * @sb: super block of the fs to be resized * @n_blocks_count: the number of blocks resides in the resized fs */ int ext4_resize_fs(struct super_block *sb, ext4_fsblk_t n_blocks_count) { struct ext4_new_flex_group_data *flex_gd = NULL; struct ext4_sb_info *sbi = EXT4_SB(sb); struct ext4_super_block *es = sbi->s_es; struct buffer_head *bh; struct inode *resize_inode = NULL; ext4_grpblk_t add, offset; unsigned long n_desc_blocks; unsigned long o_desc_blocks; ext4_group_t o_group; ext4_group_t n_group; ext4_fsblk_t o_blocks_count; ext4_fsblk_t n_blocks_count_retry = 0; unsigned long last_update_time = 0; int err = 0; int meta_bg; unsigned int flexbg_size = ext4_flex_bg_size(sbi); /* See if the device is actually as big as what was requested */ bh = ext4_sb_bread(sb, n_blocks_count - 1, 0); if (IS_ERR(bh)) { ext4_warning(sb, "can't read last block, resize aborted"); return -ENOSPC; } brelse(bh); /* * For bigalloc, trim the requested size to the nearest cluster * boundary to avoid creating an unusable filesystem. We do this * silently, instead of returning an error, to avoid breaking * callers that blindly resize the filesystem to the full size of * the underlying block device. */ if (ext4_has_feature_bigalloc(sb)) n_blocks_count &= ~((1 << EXT4_CLUSTER_BITS(sb)) - 1); retry: o_blocks_count = ext4_blocks_count(es); ext4_msg(sb, KERN_INFO, "resizing filesystem from %llu " "to %llu blocks", o_blocks_count, n_blocks_count); if (n_blocks_count < o_blocks_count) { /* On-line shrinking not supported */ ext4_warning(sb, "can't shrink FS - resize aborted"); return -EINVAL; } if (n_blocks_count == o_blocks_count) /* Nothing need to do */ return 0; n_group = ext4_get_group_number(sb, n_blocks_count - 1); if (n_group >= (0xFFFFFFFFUL / EXT4_INODES_PER_GROUP(sb))) { ext4_warning(sb, "resize would cause inodes_count overflow"); return -EINVAL; } ext4_get_group_no_and_offset(sb, o_blocks_count - 1, &o_group, &offset); n_desc_blocks = num_desc_blocks(sb, n_group + 1); o_desc_blocks = num_desc_blocks(sb, sbi->s_groups_count); meta_bg = ext4_has_feature_meta_bg(sb); if (ext4_has_feature_resize_inode(sb)) { if (meta_bg) { ext4_error(sb, "resize_inode and meta_bg enabled " "simultaneously"); return -EINVAL; } if (n_desc_blocks > o_desc_blocks + le16_to_cpu(es->s_reserved_gdt_blocks)) { n_blocks_count_retry = n_blocks_count; n_desc_blocks = o_desc_blocks + le16_to_cpu(es->s_reserved_gdt_blocks); n_group = n_desc_blocks * EXT4_DESC_PER_BLOCK(sb); n_blocks_count = (ext4_fsblk_t)n_group * EXT4_BLOCKS_PER_GROUP(sb) + le32_to_cpu(es->s_first_data_block); n_group--; /* set to last group number */ } if (!resize_inode) resize_inode = ext4_iget(sb, EXT4_RESIZE_INO, EXT4_IGET_SPECIAL); if (IS_ERR(resize_inode)) { ext4_warning(sb, "Error opening resize inode"); return PTR_ERR(resize_inode); } } if ((!resize_inode && !meta_bg && n_desc_blocks > o_desc_blocks) || n_blocks_count == o_blocks_count) { err = ext4_convert_meta_bg(sb, resize_inode); if (err) goto out; if (resize_inode) { iput(resize_inode); resize_inode = NULL; } if (n_blocks_count_retry) { n_blocks_count = n_blocks_count_retry; n_blocks_count_retry = 0; goto retry; } } /* * Make sure the last group has enough space so that it's * guaranteed to have enough space for all metadata blocks * that it might need to hold. (We might not need to store * the inode table blocks in the last block group, but there * will be cases where this might be needed.) */ if ((ext4_group_first_block_no(sb, n_group) + ext4_group_overhead_blocks(sb, n_group) + 2 + sbi->s_itb_per_group + sbi->s_cluster_ratio) >= n_blocks_count) { n_blocks_count = ext4_group_first_block_no(sb, n_group); n_group--; n_blocks_count_retry = 0; if (resize_inode) { iput(resize_inode); resize_inode = NULL; } goto retry; } /* extend the last group */ if (n_group == o_group) add = n_blocks_count - o_blocks_count; else add = EXT4_C2B(sbi, EXT4_CLUSTERS_PER_GROUP(sb) - (offset + 1)); if (add > 0) { err = ext4_group_extend_no_check(sb, o_blocks_count, add); if (err) goto out; } if (ext4_blocks_count(es) == n_blocks_count && n_blocks_count_retry == 0) goto out; err = ext4_alloc_flex_bg_array(sb, n_group + 1); if (err) goto out; err = ext4_mb_alloc_groupinfo(sb, n_group + 1); if (err) goto out; flex_gd = alloc_flex_gd(flexbg_size, o_group, n_group); if (flex_gd == NULL) { err = -ENOMEM; goto out; } /* Add flex groups. Note that a regular group is a * flex group with 1 group. */ while (ext4_setup_next_flex_gd(sb, flex_gd, n_blocks_count)) { if (time_is_before_jiffies(last_update_time + HZ * 10)) { if (last_update_time) ext4_msg(sb, KERN_INFO, "resized to %llu blocks", ext4_blocks_count(es)); last_update_time = jiffies; } if (ext4_alloc_group_tables(sb, flex_gd, flexbg_size) != 0) break; err = ext4_flex_group_add(sb, resize_inode, flex_gd); if (unlikely(err)) break; } if (!err && n_blocks_count_retry) { n_blocks_count = n_blocks_count_retry; n_blocks_count_retry = 0; free_flex_gd(flex_gd); flex_gd = NULL; if (resize_inode) { iput(resize_inode); resize_inode = NULL; } goto retry; } out: if (flex_gd) free_flex_gd(flex_gd); if (resize_inode != NULL) iput(resize_inode); if (err) ext4_warning(sb, "error (%d) occurred during " "file system resize", err); ext4_msg(sb, KERN_INFO, "resized filesystem to %llu", ext4_blocks_count(es)); return err; } |
| 175 810 111 10 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _LINUX_RANDOM_H #define _LINUX_RANDOM_H #include <linux/bug.h> #include <linux/kernel.h> #include <linux/list.h> #include <uapi/linux/random.h> struct notifier_block; void add_device_randomness(const void *buf, size_t len); void __init add_bootloader_randomness(const void *buf, size_t len); void add_input_randomness(unsigned int type, unsigned int code, unsigned int value) __latent_entropy; void add_interrupt_randomness(int irq) __latent_entropy; void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy, bool sleep_after); static inline void add_latent_entropy(void) { #if defined(LATENT_ENTROPY_PLUGIN) && !defined(__CHECKER__) add_device_randomness((const void *)&latent_entropy, sizeof(latent_entropy)); #else add_device_randomness(NULL, 0); #endif } #if IS_ENABLED(CONFIG_VMGENID) void add_vmfork_randomness(const void *unique_vm_id, size_t len); int register_random_vmfork_notifier(struct notifier_block *nb); int unregister_random_vmfork_notifier(struct notifier_block *nb); #else static inline int register_random_vmfork_notifier(struct notifier_block *nb) { return 0; } static inline int unregister_random_vmfork_notifier(struct notifier_block *nb) { return 0; } #endif void get_random_bytes(void *buf, size_t len); u8 get_random_u8(void); u16 get_random_u16(void); u32 get_random_u32(void); u64 get_random_u64(void); static inline unsigned long get_random_long(void) { #if BITS_PER_LONG == 64 return get_random_u64(); #else return get_random_u32(); #endif } u32 __get_random_u32_below(u32 ceil); /* * Returns a random integer in the interval [0, ceil), with uniform * distribution, suitable for all uses. Fastest when ceil is a constant, but * still fast for variable ceil as well. */ static inline u32 get_random_u32_below(u32 ceil) { if (!__builtin_constant_p(ceil)) return __get_random_u32_below(ceil); /* * For the fast path, below, all operations on ceil are precomputed by * the compiler, so this incurs no overhead for checking pow2, doing * divisions, or branching based on integer size. The resultant * algorithm does traditional reciprocal multiplication (typically * optimized by the compiler into shifts and adds), rejecting samples * whose lower half would indicate a range indivisible by ceil. */ BUILD_BUG_ON_MSG(!ceil, "get_random_u32_below() must take ceil > 0"); if (ceil <= 1) return 0; for (;;) { if (ceil <= 1U << 8) { u32 mult = ceil * get_random_u8(); if (likely(is_power_of_2(ceil) || (u8)mult >= (1U << 8) % ceil)) return mult >> 8; } else if (ceil <= 1U << 16) { u32 mult = ceil * get_random_u16(); if (likely(is_power_of_2(ceil) || (u16)mult >= (1U << 16) % ceil)) return mult >> 16; } else { u64 mult = (u64)ceil * get_random_u32(); if (likely(is_power_of_2(ceil) || (u32)mult >= -ceil % ceil)) return mult >> 32; } } } /* * Returns a random integer in the interval (floor, U32_MAX], with uniform * distribution, suitable for all uses. Fastest when floor is a constant, but * still fast for variable floor as well. */ static inline u32 get_random_u32_above(u32 floor) { BUILD_BUG_ON_MSG(__builtin_constant_p(floor) && floor == U32_MAX, "get_random_u32_above() must take floor < U32_MAX"); return floor + 1 + get_random_u32_below(U32_MAX - floor); } /* * Returns a random integer in the interval [floor, ceil], with uniform * distribution, suitable for all uses. Fastest when floor and ceil are * constant, but still fast for variable floor and ceil as well. */ static inline u32 get_random_u32_inclusive(u32 floor, u32 ceil) { BUILD_BUG_ON_MSG(__builtin_constant_p(floor) && __builtin_constant_p(ceil) && (floor > ceil || ceil - floor == U32_MAX), "get_random_u32_inclusive() must take floor <= ceil"); return floor + get_random_u32_below(ceil - floor + 1); } void __init random_init_early(const char *command_line); void __init random_init(void); bool rng_is_initialized(void); int wait_for_random_bytes(void); int execute_with_initialized_rng(struct notifier_block *nb); /* Calls wait_for_random_bytes() and then calls get_random_bytes(buf, nbytes). * Returns the result of the call to wait_for_random_bytes. */ static inline int get_random_bytes_wait(void *buf, size_t nbytes) { int ret = wait_for_random_bytes(); get_random_bytes(buf, nbytes); return ret; } #define declare_get_random_var_wait(name, ret_type) \ static inline int get_random_ ## name ## _wait(ret_type *out) { \ int ret = wait_for_random_bytes(); \ if (unlikely(ret)) \ return ret; \ *out = get_random_ ## name(); \ return 0; \ } declare_get_random_var_wait(u8, u8) declare_get_random_var_wait(u16, u16) declare_get_random_var_wait(u32, u32) declare_get_random_var_wait(u64, u32) declare_get_random_var_wait(long, unsigned long) #undef declare_get_random_var #ifdef CONFIG_SMP int random_prepare_cpu(unsigned int cpu); int random_online_cpu(unsigned int cpu); #endif #ifndef MODULE extern const struct file_operations random_fops, urandom_fops; #endif #endif /* _LINUX_RANDOM_H */ |
| 25 1 7 2 8 116 112 15 15 2 62 23 27 7 16 4 83 13 95 37 56 144 19 106 7 110 2 10 4 3 3 100 1 9 11 2 26 8 129 100 99 10 104 7 8 90 20 104 115 10 97 120 4 8 1 9 9 3 7 18 2 2 1 12 7 31 84 80 60 1 26 83 13 1 7 3 81 7 1 198 12 2 44 72 3 2 13 33 28 48 3 14 14 3 3 3 1 1 8 6 13 9 14 3 7 7 7 7 3 2 9 4 5 4 5 7 17 23 12 13 13 2 11 7 4 13 3 10 11 11 2 68 57 2 10 2 21 25 34 34 25 1 1 13 12 12 12 7 12 2 5 2 3 1 1 2 90 10 4 4 57 5 61 7 49 2 5 10 64 3 5 1 2 2 4 5 8 22 20 5 5 18 8 14 18 18 22 1 2 1 4 4 4 104 105 105 3 98 10 1 90 21 126 21 4 4 105 1 105 69 37 125 4 35 126 105 27 124 2 21 3 124 125 126 125 62 7 62 125 1 125 1 127 113 111 2 82 21 27 77 89 19 25 15 2 65 29 75 18 88 6 54 98 41 108 39 69 69 68 64 58 36 6 17 18 18 8 8 8 5 7 6 2 5 1 45 11 1 4 11 43 3 42 13 2 29 37 2 38 30 1 2 1 4 5 25 19 4 16 19 5 8 4 18 11 160 159 1 160 7 160 119 132 97 14 17 92 30 20 110 14 24 25 2 64 64 64 64 61 64 7 1 62 1 147 3 1 1 142 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 | // SPDX-License-Identifier: GPL-1.0+ /* * n_tty.c --- implements the N_TTY line discipline. * * This code used to be in tty_io.c, but things are getting hairy * enough that it made sense to split things off. (The N_TTY * processing has changed so much that it's hardly recognizable, * anyway...) * * Note that the open routine for N_TTY is guaranteed never to return * an error. This is because Linux will fall back to setting a line * to N_TTY if it can not switch to any other line discipline. * * Written by Theodore Ts'o, Copyright 1994. * * This file also contains code originally written by Linus Torvalds, * Copyright 1991, 1992, 1993, and by Julian Cowley, Copyright 1994. * * Reduced memory usage for older ARM systems - Russell King. * * 2000/01/20 Fixed SMP locking on put_tty_queue using bits of * the patch by Andrew J. Kroll <ag784@freenet.buffalo.edu> * who actually finally proved there really was a race. * * 2002/03/18 Implemented n_tty_wakeup to send SIGIO POLL_OUTs to * waiting writing processes-Sapan Bhatia <sapan@corewars.org>. * Also fixed a bug in BLOCKING mode where n_tty_write returns * EAGAIN */ #include <linux/bitmap.h> #include <linux/bitops.h> #include <linux/ctype.h> #include <linux/errno.h> #include <linux/export.h> #include <linux/fcntl.h> #include <linux/file.h> #include <linux/jiffies.h> #include <linux/math.h> #include <linux/poll.h> #include <linux/ratelimit.h> #include <linux/sched.h> #include <linux/signal.h> #include <linux/slab.h> #include <linux/string.h> #include <linux/tty.h> #include <linux/types.h> #include <linux/uaccess.h> #include <linux/vmalloc.h> #include "tty.h" /* * Until this number of characters is queued in the xmit buffer, select will * return "we have room for writes". */ #define WAKEUP_CHARS 256 /* * This defines the low- and high-watermarks for throttling and * unthrottling the TTY driver. These watermarks are used for * controlling the space in the read buffer. */ #define TTY_THRESHOLD_THROTTLE 128 /* now based on remaining room */ #define TTY_THRESHOLD_UNTHROTTLE 128 /* * Special byte codes used in the echo buffer to represent operations * or special handling of characters. Bytes in the echo buffer that * are not part of such special blocks are treated as normal character * codes. */ #define ECHO_OP_START 0xff #define ECHO_OP_MOVE_BACK_COL 0x80 #define ECHO_OP_SET_CANON_COL 0x81 #define ECHO_OP_ERASE_TAB 0x82 #define ECHO_COMMIT_WATERMARK 256 #define ECHO_BLOCK 256 #define ECHO_DISCARD_WATERMARK N_TTY_BUF_SIZE - (ECHO_BLOCK + 32) #undef N_TTY_TRACE #ifdef N_TTY_TRACE # define n_tty_trace(f, args...) trace_printk(f, ##args) #else # define n_tty_trace(f, args...) no_printk(f, ##args) #endif struct n_tty_data { /* producer-published */ size_t read_head; size_t commit_head; size_t canon_head; size_t echo_head; size_t echo_commit; size_t echo_mark; DECLARE_BITMAP(char_map, 256); /* private to n_tty_receive_overrun (single-threaded) */ unsigned long overrun_time; unsigned int num_overrun; /* non-atomic */ bool no_room; /* must hold exclusive termios_rwsem to reset these */ unsigned char lnext:1, erasing:1, raw:1, real_raw:1, icanon:1; unsigned char push:1; /* shared by producer and consumer */ u8 read_buf[N_TTY_BUF_SIZE]; DECLARE_BITMAP(read_flags, N_TTY_BUF_SIZE); u8 echo_buf[N_TTY_BUF_SIZE]; /* consumer-published */ size_t read_tail; size_t line_start; /* # of chars looked ahead (to find software flow control chars) */ size_t lookahead_count; /* protected by output lock */ unsigned int column; unsigned int canon_column; size_t echo_tail; struct mutex atomic_read_lock; struct mutex output_lock; }; #define MASK(x) ((x) & (N_TTY_BUF_SIZE - 1)) static inline size_t read_cnt(struct n_tty_data *ldata) { return ldata->read_head - ldata->read_tail; } static inline u8 read_buf(struct n_tty_data *ldata, size_t i) { return ldata->read_buf[MASK(i)]; } static inline u8 *read_buf_addr(struct n_tty_data *ldata, size_t i) { return &ldata->read_buf[MASK(i)]; } static inline u8 echo_buf(struct n_tty_data *ldata, size_t i) { smp_rmb(); /* Matches smp_wmb() in add_echo_byte(). */ return ldata->echo_buf[MASK(i)]; } static inline u8 *echo_buf_addr(struct n_tty_data *ldata, size_t i) { return &ldata->echo_buf[MASK(i)]; } /* If we are not echoing the data, perhaps this is a secret so erase it */ static void zero_buffer(const struct tty_struct *tty, u8 *buffer, size_t size) { if (L_ICANON(tty) && !L_ECHO(tty)) memset(buffer, 0, size); } static void tty_copy(const struct tty_struct *tty, void *to, size_t tail, size_t n) { struct n_tty_data *ldata = tty->disc_data; size_t size = N_TTY_BUF_SIZE - tail; void *from = read_buf_addr(ldata, tail); if (n > size) { tty_audit_add_data(tty, from, size); memcpy(to, from, size); zero_buffer(tty, from, size); to += size; n -= size; from = ldata->read_buf; } tty_audit_add_data(tty, from, n); memcpy(to, from, n); zero_buffer(tty, from, n); } /** * n_tty_kick_worker - start input worker (if required) * @tty: terminal * * Re-schedules the flip buffer work if it may have stopped. * * Locking: * * Caller holds exclusive %termios_rwsem, or * * n_tty_read()/consumer path: * holds non-exclusive %termios_rwsem */ static void n_tty_kick_worker(const struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; /* Did the input worker stop? Restart it */ if (unlikely(READ_ONCE(ldata->no_room))) { WRITE_ONCE(ldata->no_room, 0); WARN_RATELIMIT(tty->port->itty == NULL, "scheduling with invalid itty\n"); /* see if ldisc has been killed - if so, this means that * even though the ldisc has been halted and ->buf.work * cancelled, ->buf.work is about to be rescheduled */ WARN_RATELIMIT(test_bit(TTY_LDISC_HALTED, &tty->flags), "scheduling buffer work for halted ldisc\n"); tty_buffer_restart_work(tty->port); } } static ssize_t chars_in_buffer(const struct tty_struct *tty) { const struct n_tty_data *ldata = tty->disc_data; size_t head = ldata->icanon ? ldata->canon_head : ldata->commit_head; return head - ldata->read_tail; } /** * n_tty_write_wakeup - asynchronous I/O notifier * @tty: tty device * * Required for the ptys, serial driver etc. since processes that attach * themselves to the master and rely on ASYNC IO must be woken up. */ static void n_tty_write_wakeup(struct tty_struct *tty) { clear_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); kill_fasync(&tty->fasync, SIGIO, POLL_OUT); } static void n_tty_check_throttle(struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; /* * Check the remaining room for the input canonicalization * mode. We don't want to throttle the driver if we're in * canonical mode and don't have a newline yet! */ if (ldata->icanon && ldata->canon_head == ldata->read_tail) return; do { tty_set_flow_change(tty, TTY_THROTTLE_SAFE); if (N_TTY_BUF_SIZE - read_cnt(ldata) >= TTY_THRESHOLD_THROTTLE) break; } while (!tty_throttle_safe(tty)); __tty_set_flow_change(tty, 0); } static void n_tty_check_unthrottle(struct tty_struct *tty) { if (tty->driver->type == TTY_DRIVER_TYPE_PTY) { if (chars_in_buffer(tty) > TTY_THRESHOLD_UNTHROTTLE) return; n_tty_kick_worker(tty); tty_wakeup(tty->link); return; } /* If there is enough space in the read buffer now, let the * low-level driver know. We use chars_in_buffer() to * check the buffer, as it now knows about canonical mode. * Otherwise, if the driver is throttled and the line is * longer than TTY_THRESHOLD_UNTHROTTLE in canonical mode, * we won't get any more characters. */ do { tty_set_flow_change(tty, TTY_UNTHROTTLE_SAFE); if (chars_in_buffer(tty) > TTY_THRESHOLD_UNTHROTTLE) break; n_tty_kick_worker(tty); } while (!tty_unthrottle_safe(tty)); __tty_set_flow_change(tty, 0); } /** * put_tty_queue - add character to tty * @c: character * @ldata: n_tty data * * Add a character to the tty read_buf queue. * * Locking: * * n_tty_receive_buf()/producer path: * caller holds non-exclusive %termios_rwsem */ static inline void put_tty_queue(u8 c, struct n_tty_data *ldata) { *read_buf_addr(ldata, ldata->read_head) = c; ldata->read_head++; } /** * reset_buffer_flags - reset buffer state * @ldata: line disc data to reset * * Reset the read buffer counters and clear the flags. Called from * n_tty_open() and n_tty_flush_buffer(). * * Locking: * * caller holds exclusive %termios_rwsem, or * * (locking is not required) */ static void reset_buffer_flags(struct n_tty_data *ldata) { ldata->read_head = ldata->canon_head = ldata->read_tail = 0; ldata->commit_head = 0; ldata->line_start = 0; ldata->erasing = 0; bitmap_zero(ldata->read_flags, N_TTY_BUF_SIZE); ldata->push = 0; ldata->lookahead_count = 0; } static void n_tty_packet_mode_flush(struct tty_struct *tty) { unsigned long flags; if (tty->link->ctrl.packet) { spin_lock_irqsave(&tty->ctrl.lock, flags); tty->ctrl.pktstatus |= TIOCPKT_FLUSHREAD; spin_unlock_irqrestore(&tty->ctrl.lock, flags); wake_up_interruptible(&tty->link->read_wait); } } /** * n_tty_flush_buffer - clean input queue * @tty: terminal device * * Flush the input buffer. Called when the tty layer wants the buffer flushed * (eg at hangup) or when the %N_TTY line discipline internally has to clean * the pending queue (for example some signals). * * Holds %termios_rwsem to exclude producer/consumer while buffer indices are * reset. * * Locking: %ctrl.lock, exclusive %termios_rwsem */ static void n_tty_flush_buffer(struct tty_struct *tty) { down_write(&tty->termios_rwsem); reset_buffer_flags(tty->disc_data); n_tty_kick_worker(tty); if (tty->link) n_tty_packet_mode_flush(tty); up_write(&tty->termios_rwsem); } /** * is_utf8_continuation - utf8 multibyte check * @c: byte to check * * Returns: true if the utf8 character @c is a multibyte continuation * character. We use this to correctly compute the on-screen size of the * character when printing. */ static inline int is_utf8_continuation(u8 c) { return (c & 0xc0) == 0x80; } /** * is_continuation - multibyte check * @c: byte to check * @tty: terminal device * * Returns: true if the utf8 character @c is a multibyte continuation character * and the terminal is in unicode mode. */ static inline int is_continuation(u8 c, const struct tty_struct *tty) { return I_IUTF8(tty) && is_utf8_continuation(c); } /** * do_output_char - output one character * @c: character (or partial unicode symbol) * @tty: terminal device * @space: space available in tty driver write buffer * * This is a helper function that handles one output character (including * special characters like TAB, CR, LF, etc.), doing OPOST processing and * putting the results in the tty driver's write buffer. * * Note that Linux currently ignores TABDLY, CRDLY, VTDLY, FFDLY and NLDLY. * They simply aren't relevant in the world today. If you ever need them, add * them here. * * Returns: the number of bytes of buffer space used or -1 if no space left. * * Locking: should be called under the %output_lock to protect the column state * and space left in the buffer. */ static int do_output_char(u8 c, struct tty_struct *tty, int space) { struct n_tty_data *ldata = tty->disc_data; int spaces; if (!space) return -1; switch (c) { case '\n': if (O_ONLRET(tty)) ldata->column = 0; if (O_ONLCR(tty)) { if (space < 2) return -1; ldata->canon_column = ldata->column = 0; tty->ops->write(tty, "\r\n", 2); return 2; } ldata->canon_column = ldata->column; break; case '\r': if (O_ONOCR(tty) && ldata->column == 0) return 0; if (O_OCRNL(tty)) { c = '\n'; if (O_ONLRET(tty)) ldata->canon_column = ldata->column = 0; break; } ldata->canon_column = ldata->column = 0; break; case '\t': spaces = 8 - (ldata->column & 7); if (O_TABDLY(tty) == XTABS) { if (space < spaces) return -1; ldata->column += spaces; tty->ops->write(tty, " ", spaces); return spaces; } ldata->column += spaces; break; case '\b': if (ldata->column > 0) ldata->column--; break; default: if (!iscntrl(c)) { if (O_OLCUC(tty)) c = toupper(c); if (!is_continuation(c, tty)) ldata->column++; } break; } tty_put_char(tty, c); return 1; } /** * process_output - output post processor * @c: character (or partial unicode symbol) * @tty: terminal device * * Output one character with OPOST processing. * * Returns: -1 when the output device is full and the character must be * retried. * * Locking: %output_lock to protect column state and space left (also, this is *called from n_tty_write() under the tty layer write lock). */ static int process_output(u8 c, struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; int space, retval; mutex_lock(&ldata->output_lock); space = tty_write_room(tty); retval = do_output_char(c, tty, space); mutex_unlock(&ldata->output_lock); if (retval < 0) return -1; else return 0; } /** * process_output_block - block post processor * @tty: terminal device * @buf: character buffer * @nr: number of bytes to output * * Output a block of characters with OPOST processing. * * This path is used to speed up block console writes, among other things when * processing blocks of output data. It handles only the simple cases normally * found and helps to generate blocks of symbols for the console driver and * thus improve performance. * * Returns: the number of characters output. * * Locking: %output_lock to protect column state and space left (also, this is * called from n_tty_write() under the tty layer write lock). */ static ssize_t process_output_block(struct tty_struct *tty, const u8 *buf, unsigned int nr) { struct n_tty_data *ldata = tty->disc_data; int space; int i; const u8 *cp; mutex_lock(&ldata->output_lock); space = tty_write_room(tty); if (space <= 0) { mutex_unlock(&ldata->output_lock); return space; } if (nr > space) nr = space; for (i = 0, cp = buf; i < nr; i++, cp++) { u8 c = *cp; switch (c) { case '\n': if (O_ONLRET(tty)) ldata->column = 0; if (O_ONLCR(tty)) goto break_out; ldata->canon_column = ldata->column; break; case '\r': if (O_ONOCR(tty) && ldata->column == 0) goto break_out; if (O_OCRNL(tty)) goto break_out; ldata->canon_column = ldata->column = 0; break; case '\t': goto break_out; case '\b': if (ldata->column > 0) ldata->column--; break; default: if (!iscntrl(c)) { if (O_OLCUC(tty)) goto break_out; if (!is_continuation(c, tty)) ldata->column++; } break; } } break_out: i = tty->ops->write(tty, buf, i); mutex_unlock(&ldata->output_lock); return i; } static int n_tty_process_echo_ops(struct tty_struct *tty, size_t *tail, int space) { struct n_tty_data *ldata = tty->disc_data; u8 op; /* * Since add_echo_byte() is called without holding output_lock, we * might see only portion of multi-byte operation. */ if (MASK(ldata->echo_commit) == MASK(*tail + 1)) return -ENODATA; /* * If the buffer byte is the start of a multi-byte operation, get the * next byte, which is either the op code or a control character value. */ op = echo_buf(ldata, *tail + 1); switch (op) { case ECHO_OP_ERASE_TAB: { unsigned int num_chars, num_bs; if (MASK(ldata->echo_commit) == MASK(*tail + 2)) return -ENODATA; num_chars = echo_buf(ldata, *tail + 2); /* * Determine how many columns to go back in order to erase the * tab. This depends on the number of columns used by other * characters within the tab area. If this (modulo 8) count is * from the start of input rather than from a previous tab, we * offset by canon column. Otherwise, tab spacing is normal. */ if (!(num_chars & 0x80)) num_chars += ldata->canon_column; num_bs = 8 - (num_chars & 7); if (num_bs > space) return -ENOSPC; space -= num_bs; while (num_bs--) { tty_put_char(tty, '\b'); if (ldata->column > 0) ldata->column--; } *tail += 3; break; } case ECHO_OP_SET_CANON_COL: ldata->canon_column = ldata->column; *tail += 2; break; case ECHO_OP_MOVE_BACK_COL: if (ldata->column > 0) ldata->column--; *tail += 2; break; case ECHO_OP_START: /* This is an escaped echo op start code */ if (!space) return -ENOSPC; tty_put_char(tty, ECHO_OP_START); ldata->column++; space--; *tail += 2; break; default: /* * If the op is not a special byte code, it is a ctrl char * tagged to be echoed as "^X" (where X is the letter * representing the control char). Note that we must ensure * there is enough space for the whole ctrl pair. */ if (space < 2) return -ENOSPC; tty_put_char(tty, '^'); tty_put_char(tty, op ^ 0100); ldata->column += 2; space -= 2; *tail += 2; break; } return space; } /** * __process_echoes - write pending echo characters * @tty: terminal device * * Write previously buffered echo (and other ldisc-generated) characters to the * tty. * * Characters generated by the ldisc (including echoes) need to be buffered * because the driver's write buffer can fill during heavy program output. * Echoing straight to the driver will often fail under these conditions, * causing lost characters and resulting mismatches of ldisc state information. * * Since the ldisc state must represent the characters actually sent to the * driver at the time of the write, operations like certain changes in column * state are also saved in the buffer and executed here. * * A circular fifo buffer is used so that the most recent characters are * prioritized. Also, when control characters are echoed with a prefixed "^", * the pair is treated atomically and thus not separated. * * Locking: callers must hold %output_lock. */ static size_t __process_echoes(struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; int space, old_space; size_t tail; u8 c; old_space = space = tty_write_room(tty); tail = ldata->echo_tail; while (MASK(ldata->echo_commit) != MASK(tail)) { c = echo_buf(ldata, tail); if (c == ECHO_OP_START) { int ret = n_tty_process_echo_ops(tty, &tail, space); if (ret == -ENODATA) goto not_yet_stored; if (ret < 0) break; space = ret; } else { if (O_OPOST(tty)) { int retval = do_output_char(c, tty, space); if (retval < 0) break; space -= retval; } else { if (!space) break; tty_put_char(tty, c); space -= 1; } tail += 1; } } /* If the echo buffer is nearly full (so that the possibility exists * of echo overrun before the next commit), then discard enough * data at the tail to prevent a subsequent overrun */ while (ldata->echo_commit > tail && ldata->echo_commit - tail >= ECHO_DISCARD_WATERMARK) { if (echo_buf(ldata, tail) == ECHO_OP_START) { if (echo_buf(ldata, tail + 1) == ECHO_OP_ERASE_TAB) tail += 3; else tail += 2; } else tail++; } not_yet_stored: ldata->echo_tail = tail; return old_space - space; } static void commit_echoes(struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; size_t nr, old, echoed; size_t head; mutex_lock(&ldata->output_lock); head = ldata->echo_head; ldata->echo_mark = head; old = ldata->echo_commit - ldata->echo_tail; /* Process committed echoes if the accumulated # of bytes * is over the threshold (and try again each time another * block is accumulated) */ nr = head - ldata->echo_tail; if (nr < ECHO_COMMIT_WATERMARK || (nr % ECHO_BLOCK > old % ECHO_BLOCK)) { mutex_unlock(&ldata->output_lock); return; } ldata->echo_commit = head; echoed = __process_echoes(tty); mutex_unlock(&ldata->output_lock); if (echoed && tty->ops->flush_chars) tty->ops->flush_chars(tty); } static void process_echoes(struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; size_t echoed; if (ldata->echo_mark == ldata->echo_tail) return; mutex_lock(&ldata->output_lock); ldata->echo_commit = ldata->echo_mark; echoed = __process_echoes(tty); mutex_unlock(&ldata->output_lock); if (echoed && tty->ops->flush_chars) tty->ops->flush_chars(tty); } /* NB: echo_mark and echo_head should be equivalent here */ static void flush_echoes(struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; if ((!L_ECHO(tty) && !L_ECHONL(tty)) || ldata->echo_commit == ldata->echo_head) return; mutex_lock(&ldata->output_lock); ldata->echo_commit = ldata->echo_head; __process_echoes(tty); mutex_unlock(&ldata->output_lock); } /** * add_echo_byte - add a byte to the echo buffer * @c: unicode byte to echo * @ldata: n_tty data * * Add a character or operation byte to the echo buffer. */ static inline void add_echo_byte(u8 c, struct n_tty_data *ldata) { *echo_buf_addr(ldata, ldata->echo_head) = c; smp_wmb(); /* Matches smp_rmb() in echo_buf(). */ ldata->echo_head++; } /** * echo_move_back_col - add operation to move back a column * @ldata: n_tty data * * Add an operation to the echo buffer to move back one column. */ static void echo_move_back_col(struct n_tty_data *ldata) { add_echo_byte(ECHO_OP_START, ldata); add_echo_byte(ECHO_OP_MOVE_BACK_COL, ldata); } /** * echo_set_canon_col - add operation to set the canon column * @ldata: n_tty data * * Add an operation to the echo buffer to set the canon column to the current * column. */ static void echo_set_canon_col(struct n_tty_data *ldata) { add_echo_byte(ECHO_OP_START, ldata); add_echo_byte(ECHO_OP_SET_CANON_COL, ldata); } /** * echo_erase_tab - add operation to erase a tab * @num_chars: number of character columns already used * @after_tab: true if num_chars starts after a previous tab * @ldata: n_tty data * * Add an operation to the echo buffer to erase a tab. * * Called by the eraser function, which knows how many character columns have * been used since either a previous tab or the start of input. This * information will be used later, along with canon column (if applicable), to * go back the correct number of columns. */ static void echo_erase_tab(unsigned int num_chars, int after_tab, struct n_tty_data *ldata) { add_echo_byte(ECHO_OP_START, ldata); add_echo_byte(ECHO_OP_ERASE_TAB, ldata); /* We only need to know this modulo 8 (tab spacing) */ num_chars &= 7; /* Set the high bit as a flag if num_chars is after a previous tab */ if (after_tab) num_chars |= 0x80; add_echo_byte(num_chars, ldata); } /** * echo_char_raw - echo a character raw * @c: unicode byte to echo * @ldata: line disc data * * Echo user input back onto the screen. This must be called only when * L_ECHO(tty) is true. Called from the &tty_driver.receive_buf() path. * * This variant does not treat control characters specially. */ static void echo_char_raw(u8 c, struct n_tty_data *ldata) { if (c == ECHO_OP_START) { add_echo_byte(ECHO_OP_START, ldata); add_echo_byte(ECHO_OP_START, ldata); } else { add_echo_byte(c, ldata); } } /** * echo_char - echo a character * @c: unicode byte to echo * @tty: terminal device * * Echo user input back onto the screen. This must be called only when * L_ECHO(tty) is true. Called from the &tty_driver.receive_buf() path. * * This variant tags control characters to be echoed as "^X" (where X is the * letter representing the control char). */ static void echo_char(u8 c, const struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; if (c == ECHO_OP_START) { add_echo_byte(ECHO_OP_START, ldata); add_echo_byte(ECHO_OP_START, ldata); } else { if (L_ECHOCTL(tty) && iscntrl(c) && c != '\t') add_echo_byte(ECHO_OP_START, ldata); add_echo_byte(c, ldata); } } /** * finish_erasing - complete erase * @ldata: n_tty data */ static inline void finish_erasing(struct n_tty_data *ldata) { if (ldata->erasing) { echo_char_raw('/', ldata); ldata->erasing = 0; } } /** * eraser - handle erase function * @c: character input * @tty: terminal device * * Perform erase and necessary output when an erase character is present in the * stream from the driver layer. Handles the complexities of UTF-8 multibyte * symbols. * * Locking: n_tty_receive_buf()/producer path: * caller holds non-exclusive %termios_rwsem */ static void eraser(u8 c, const struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; enum { ERASE, WERASE, KILL } kill_type; size_t head; size_t cnt; int seen_alnums; if (ldata->read_head == ldata->canon_head) { /* process_output('\a', tty); */ /* what do you think? */ return; } if (c == ERASE_CHAR(tty)) kill_type = ERASE; else if (c == WERASE_CHAR(tty)) kill_type = WERASE; else { if (!L_ECHO(tty)) { ldata->read_head = ldata->canon_head; return; } if (!L_ECHOK(tty) || !L_ECHOKE(tty) || !L_ECHOE(tty)) { ldata->read_head = ldata->canon_head; finish_erasing(ldata); echo_char(KILL_CHAR(tty), tty); /* Add a newline if ECHOK is on and ECHOKE is off. */ if (L_ECHOK(tty)) echo_char_raw('\n', ldata); return; } kill_type = KILL; } seen_alnums = 0; while (MASK(ldata->read_head) != MASK(ldata->canon_head)) { head = ldata->read_head; /* erase a single possibly multibyte character */ do { head--; c = read_buf(ldata, head); } while (is_continuation(c, tty) && MASK(head) != MASK(ldata->canon_head)); /* do not partially erase */ if (is_continuation(c, tty)) break; if (kill_type == WERASE) { /* Equivalent to BSD's ALTWERASE. */ if (isalnum(c) || c == '_') seen_alnums++; else if (seen_alnums) break; } cnt = ldata->read_head - head; ldata->read_head = head; if (L_ECHO(tty)) { if (L_ECHOPRT(tty)) { if (!ldata->erasing) { echo_char_raw('\\', ldata); ldata->erasing = 1; } /* if cnt > 1, output a multi-byte character */ echo_char(c, tty); while (--cnt > 0) { head++; echo_char_raw(read_buf(ldata, head), ldata); echo_move_back_col(ldata); } } else if (kill_type == ERASE && !L_ECHOE(tty)) { echo_char(ERASE_CHAR(tty), tty); } else if (c == '\t') { unsigned int num_chars = 0; int after_tab = 0; size_t tail = ldata->read_head; /* * Count the columns used for characters * since the start of input or after a * previous tab. * This info is used to go back the correct * number of columns. */ while (MASK(tail) != MASK(ldata->canon_head)) { tail--; c = read_buf(ldata, tail); if (c == '\t') { after_tab = 1; break; } else if (iscntrl(c)) { if (L_ECHOCTL(tty)) num_chars += 2; } else if (!is_continuation(c, tty)) { num_chars++; } } echo_erase_tab(num_chars, after_tab, ldata); } else { if (iscntrl(c) && L_ECHOCTL(tty)) { echo_char_raw('\b', ldata); echo_char_raw(' ', ldata); echo_char_raw('\b', ldata); } if (!iscntrl(c) || L_ECHOCTL(tty)) { echo_char_raw('\b', ldata); echo_char_raw(' ', ldata); echo_char_raw('\b', ldata); } } } if (kill_type == ERASE) break; } if (ldata->read_head == ldata->canon_head && L_ECHO(tty)) finish_erasing(ldata); } static void __isig(int sig, struct tty_struct *tty) { struct pid *tty_pgrp = tty_get_pgrp(tty); if (tty_pgrp) { kill_pgrp(tty_pgrp, sig, 1); put_pid(tty_pgrp); } } /** * isig - handle the ISIG optio * @sig: signal * @tty: terminal * * Called when a signal is being sent due to terminal input. Called from the * &tty_driver.receive_buf() path, so serialized. * * Performs input and output flush if !NOFLSH. In this context, the echo * buffer is 'output'. The signal is processed first to alert any current * readers or writers to discontinue and exit their i/o loops. * * Locking: %ctrl.lock */ static void isig(int sig, struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; if (L_NOFLSH(tty)) { /* signal only */ __isig(sig, tty); } else { /* signal and flush */ up_read(&tty->termios_rwsem); down_write(&tty->termios_rwsem); __isig(sig, tty); /* clear echo buffer */ mutex_lock(&ldata->output_lock); ldata->echo_head = ldata->echo_tail = 0; ldata->echo_mark = ldata->echo_commit = 0; mutex_unlock(&ldata->output_lock); /* clear output buffer */ tty_driver_flush_buffer(tty); /* clear input buffer */ reset_buffer_flags(tty->disc_data); /* notify pty master of flush */ if (tty->link) n_tty_packet_mode_flush(tty); up_write(&tty->termios_rwsem); down_read(&tty->termios_rwsem); } } /** * n_tty_receive_break - handle break * @tty: terminal * * An RS232 break event has been hit in the incoming bitstream. This can cause * a variety of events depending upon the termios settings. * * Locking: n_tty_receive_buf()/producer path: * caller holds non-exclusive termios_rwsem * * Note: may get exclusive %termios_rwsem if flushing input buffer */ static void n_tty_receive_break(struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; if (I_IGNBRK(tty)) return; if (I_BRKINT(tty)) { isig(SIGINT, tty); return; } if (I_PARMRK(tty)) { put_tty_queue('\377', ldata); put_tty_queue('\0', ldata); } put_tty_queue('\0', ldata); } /** * n_tty_receive_overrun - handle overrun reporting * @tty: terminal * * Data arrived faster than we could process it. While the tty driver has * flagged this the bits that were missed are gone forever. * * Called from the receive_buf path so single threaded. Does not need locking * as num_overrun and overrun_time are function private. */ static void n_tty_receive_overrun(const struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; ldata->num_overrun++; if (time_is_before_jiffies(ldata->overrun_time + HZ)) { tty_warn(tty, "%u input overrun(s)\n", ldata->num_overrun); ldata->overrun_time = jiffies; ldata->num_overrun = 0; } } /** * n_tty_receive_parity_error - error notifier * @tty: terminal device * @c: character * * Process a parity error and queue the right data to indicate the error case * if necessary. * * Locking: n_tty_receive_buf()/producer path: * caller holds non-exclusive %termios_rwsem */ static void n_tty_receive_parity_error(const struct tty_struct *tty, u8 c) { struct n_tty_data *ldata = tty->disc_data; if (I_INPCK(tty)) { if (I_IGNPAR(tty)) return; if (I_PARMRK(tty)) { put_tty_queue('\377', ldata); put_tty_queue('\0', ldata); put_tty_queue(c, ldata); } else put_tty_queue('\0', ldata); } else put_tty_queue(c, ldata); } static void n_tty_receive_signal_char(struct tty_struct *tty, int signal, u8 c) { isig(signal, tty); if (I_IXON(tty)) start_tty(tty); if (L_ECHO(tty)) { echo_char(c, tty); commit_echoes(tty); } else process_echoes(tty); } static bool n_tty_is_char_flow_ctrl(struct tty_struct *tty, u8 c) { return c == START_CHAR(tty) || c == STOP_CHAR(tty); } /** * n_tty_receive_char_flow_ctrl - receive flow control chars * @tty: terminal device * @c: character * @lookahead_done: lookahead has processed this character already * * Receive and process flow control character actions. * * In case lookahead for flow control chars already handled the character in * advance to the normal receive, the actions are skipped during normal * receive. * * Returns true if @c is consumed as flow-control character, the character * must not be treated as normal character. */ static bool n_tty_receive_char_flow_ctrl(struct tty_struct *tty, u8 c, bool lookahead_done) { if (!n_tty_is_char_flow_ctrl(tty, c)) return false; if (lookahead_done) return true; if (c == START_CHAR(tty)) { start_tty(tty); process_echoes(tty); return true; } /* STOP_CHAR */ stop_tty(tty); return true; } static void n_tty_receive_handle_newline(struct tty_struct *tty, u8 c) { struct n_tty_data *ldata = tty->disc_data; set_bit(MASK(ldata->read_head), ldata->read_flags); put_tty_queue(c, ldata); smp_store_release(&ldata->canon_head, ldata->read_head); kill_fasync(&tty->fasync, SIGIO, POLL_IN); wake_up_interruptible_poll(&tty->read_wait, EPOLLIN | EPOLLRDNORM); } static bool n_tty_receive_char_canon(struct tty_struct *tty, u8 c) { struct n_tty_data *ldata = tty->disc_data; if (c == ERASE_CHAR(tty) || c == KILL_CHAR(tty) || (c == WERASE_CHAR(tty) && L_IEXTEN(tty))) { eraser(c, tty); commit_echoes(tty); return true; } if (c == LNEXT_CHAR(tty) && L_IEXTEN(tty)) { ldata->lnext = 1; if (L_ECHO(tty)) { finish_erasing(ldata); if (L_ECHOCTL(tty)) { echo_char_raw('^', ldata); echo_char_raw('\b', ldata); commit_echoes(tty); } } return true; } if (c == REPRINT_CHAR(tty) && L_ECHO(tty) && L_IEXTEN(tty)) { size_t tail = ldata->canon_head; finish_erasing(ldata); echo_char(c, tty); echo_char_raw('\n', ldata); while (MASK(tail) != MASK(ldata->read_head)) { echo_char(read_buf(ldata, tail), tty); tail++; } commit_echoes(tty); return true; } if (c == '\n') { if (L_ECHO(tty) || L_ECHONL(tty)) { echo_char_raw('\n', ldata); commit_echoes(tty); } n_tty_receive_handle_newline(tty, c); return true; } if (c == EOF_CHAR(tty)) { c = __DISABLED_CHAR; n_tty_receive_handle_newline(tty, c); return true; } if ((c == EOL_CHAR(tty)) || (c == EOL2_CHAR(tty) && L_IEXTEN(tty))) { /* * XXX are EOL_CHAR and EOL2_CHAR echoed?!? */ if (L_ECHO(tty)) { /* Record the column of first canon char. */ if (ldata->canon_head == ldata->read_head) echo_set_canon_col(ldata); echo_char(c, tty); commit_echoes(tty); } /* * XXX does PARMRK doubling happen for * EOL_CHAR and EOL2_CHAR? */ if (c == '\377' && I_PARMRK(tty)) put_tty_queue(c, ldata); n_tty_receive_handle_newline(tty, c); return true; } return false; } static void n_tty_receive_char_special(struct tty_struct *tty, u8 c, bool lookahead_done) { struct n_tty_data *ldata = tty->disc_data; if (I_IXON(tty) && n_tty_receive_char_flow_ctrl(tty, c, lookahead_done)) return; if (L_ISIG(tty)) { if (c == INTR_CHAR(tty)) { n_tty_receive_signal_char(tty, SIGINT, c); return; } else if (c == QUIT_CHAR(tty)) { n_tty_receive_signal_char(tty, SIGQUIT, c); return; } else if (c == SUSP_CHAR(tty)) { n_tty_receive_signal_char(tty, SIGTSTP, c); return; } } if (tty->flow.stopped && !tty->flow.tco_stopped && I_IXON(tty) && I_IXANY(tty)) { start_tty(tty); process_echoes(tty); } if (c == '\r') { if (I_IGNCR(tty)) return; if (I_ICRNL(tty)) c = '\n'; } else if (c == '\n' && I_INLCR(tty)) c = '\r'; if (ldata->icanon && n_tty_receive_char_canon(tty, c)) return; if (L_ECHO(tty)) { finish_erasing(ldata); if (c == '\n') echo_char_raw('\n', ldata); else { /* Record the column of first canon char. */ if (ldata->canon_head == ldata->read_head) echo_set_canon_col(ldata); echo_char(c, tty); } commit_echoes(tty); } /* PARMRK doubling check */ if (c == '\377' && I_PARMRK(tty)) put_tty_queue(c, ldata); put_tty_queue(c, ldata); } /** * n_tty_receive_char - perform processing * @tty: terminal device * @c: character * * Process an individual character of input received from the driver. This is * serialized with respect to itself by the rules for the driver above. * * Locking: n_tty_receive_buf()/producer path: * caller holds non-exclusive %termios_rwsem * publishes canon_head if canonical mode is active */ static void n_tty_receive_char(struct tty_struct *tty, u8 c) { struct n_tty_data *ldata = tty->disc_data; if (tty->flow.stopped && !tty->flow.tco_stopped && I_IXON(tty) && I_IXANY(tty)) { start_tty(tty); process_echoes(tty); } if (L_ECHO(tty)) { finish_erasing(ldata); /* Record the column of first canon char. */ if (ldata->canon_head == ldata->read_head) echo_set_canon_col(ldata); echo_char(c, tty); commit_echoes(tty); } /* PARMRK doubling check */ if (c == '\377' && I_PARMRK(tty)) put_tty_queue(c, ldata); put_tty_queue(c, ldata); } static void n_tty_receive_char_closing(struct tty_struct *tty, u8 c, bool lookahead_done) { if (I_ISTRIP(tty)) c &= 0x7f; if (I_IUCLC(tty) && L_IEXTEN(tty)) c = tolower(c); if (I_IXON(tty)) { if (!n_tty_receive_char_flow_ctrl(tty, c, lookahead_done) && tty->flow.stopped && !tty->flow.tco_stopped && I_IXANY(tty) && c != INTR_CHAR(tty) && c != QUIT_CHAR(tty) && c != SUSP_CHAR(tty)) { start_tty(tty); process_echoes(tty); } } } static void n_tty_receive_char_flagged(struct tty_struct *tty, u8 c, u8 flag) { switch (flag) { case TTY_BREAK: n_tty_receive_break(tty); break; case TTY_PARITY: case TTY_FRAME: n_tty_receive_parity_error(tty, c); break; case TTY_OVERRUN: n_tty_receive_overrun(tty); break; default: tty_err(tty, "unknown flag %u\n", flag); break; } } static void n_tty_receive_char_lnext(struct tty_struct *tty, u8 c, u8 flag) { struct n_tty_data *ldata = tty->disc_data; ldata->lnext = 0; if (likely(flag == TTY_NORMAL)) { if (I_ISTRIP(tty)) c &= 0x7f; if (I_IUCLC(tty) && L_IEXTEN(tty)) c = tolower(c); n_tty_receive_char(tty, c); } else n_tty_receive_char_flagged(tty, c, flag); } /* Caller must ensure count > 0 */ static void n_tty_lookahead_flow_ctrl(struct tty_struct *tty, const u8 *cp, const u8 *fp, size_t count) { struct n_tty_data *ldata = tty->disc_data; u8 flag = TTY_NORMAL; ldata->lookahead_count += count; if (!I_IXON(tty)) return; while (count--) { if (fp) flag = *fp++; if (likely(flag == TTY_NORMAL)) n_tty_receive_char_flow_ctrl(tty, *cp, false); cp++; } } static void n_tty_receive_buf_real_raw(const struct tty_struct *tty, const u8 *cp, size_t count) { struct n_tty_data *ldata = tty->disc_data; /* handle buffer wrap-around by a loop */ for (unsigned int i = 0; i < 2; i++) { size_t head = MASK(ldata->read_head); size_t n = min(count, N_TTY_BUF_SIZE - head); memcpy(read_buf_addr(ldata, head), cp, n); ldata->read_head += n; cp += n; count -= n; } } static void n_tty_receive_buf_raw(struct tty_struct *tty, const u8 *cp, const u8 *fp, size_t count) { struct n_tty_data *ldata = tty->disc_data; u8 flag = TTY_NORMAL; while (count--) { if (fp) flag = *fp++; if (likely(flag == TTY_NORMAL)) put_tty_queue(*cp++, ldata); else n_tty_receive_char_flagged(tty, *cp++, flag); } } static void n_tty_receive_buf_closing(struct tty_struct *tty, const u8 *cp, const u8 *fp, size_t count, bool lookahead_done) { u8 flag = TTY_NORMAL; while (count--) { if (fp) flag = *fp++; if (likely(flag == TTY_NORMAL)) n_tty_receive_char_closing(tty, *cp++, lookahead_done); } } static void n_tty_receive_buf_standard(struct tty_struct *tty, const u8 *cp, const u8 *fp, size_t count, bool lookahead_done) { struct n_tty_data *ldata = tty->disc_data; u8 flag = TTY_NORMAL; while (count--) { u8 c = *cp++; if (fp) flag = *fp++; if (ldata->lnext) { n_tty_receive_char_lnext(tty, c, flag); continue; } if (unlikely(flag != TTY_NORMAL)) { n_tty_receive_char_flagged(tty, c, flag); continue; } if (I_ISTRIP(tty)) c &= 0x7f; if (I_IUCLC(tty) && L_IEXTEN(tty)) c = tolower(c); if (L_EXTPROC(tty)) { put_tty_queue(c, ldata); continue; } if (test_bit(c, ldata->char_map)) n_tty_receive_char_special(tty, c, lookahead_done); else n_tty_receive_char(tty, c); } } static void __receive_buf(struct tty_struct *tty, const u8 *cp, const u8 *fp, size_t count) { struct n_tty_data *ldata = tty->disc_data; bool preops = I_ISTRIP(tty) || (I_IUCLC(tty) && L_IEXTEN(tty)); size_t la_count = min(ldata->lookahead_count, count); if (ldata->real_raw) n_tty_receive_buf_real_raw(tty, cp, count); else if (ldata->raw || (L_EXTPROC(tty) && !preops)) n_tty_receive_buf_raw(tty, cp, fp, count); else if (tty->closing && !L_EXTPROC(tty)) { if (la_count > 0) { n_tty_receive_buf_closing(tty, cp, fp, la_count, true); cp += la_count; if (fp) fp += la_count; count -= la_count; } if (count > 0) n_tty_receive_buf_closing(tty, cp, fp, count, false); } else { if (la_count > 0) { n_tty_receive_buf_standard(tty, cp, fp, la_count, true); cp += la_count; if (fp) fp += la_count; count -= la_count; } if (count > 0) n_tty_receive_buf_standard(tty, cp, fp, count, false); flush_echoes(tty); if (tty->ops->flush_chars) tty->ops->flush_chars(tty); } ldata->lookahead_count -= la_count; if (ldata->icanon && !L_EXTPROC(tty)) return; /* publish read_head to consumer */ smp_store_release(&ldata->commit_head, ldata->read_head); if (read_cnt(ldata)) { kill_fasync(&tty->fasync, SIGIO, POLL_IN); wake_up_interruptible_poll(&tty->read_wait, EPOLLIN | EPOLLRDNORM); } } /** * n_tty_receive_buf_common - process input * @tty: device to receive input * @cp: input chars * @fp: flags for each char (if %NULL, all chars are %TTY_NORMAL) * @count: number of input chars in @cp * @flow: enable flow control * * Called by the terminal driver when a block of characters has been received. * This function must be called from soft contexts not from interrupt context. * The driver is responsible for making calls one at a time and in order (or * using flush_to_ldisc()). * * Returns: the # of input chars from @cp which were processed. * * In canonical mode, the maximum line length is 4096 chars (including the line * termination char); lines longer than 4096 chars are truncated. After 4095 * chars, input data is still processed but not stored. Overflow processing * ensures the tty can always receive more input until at least one line can be * read. * * In non-canonical mode, the read buffer will only accept 4095 chars; this * provides the necessary space for a newline char if the input mode is * switched to canonical. * * Note it is possible for the read buffer to _contain_ 4096 chars in * non-canonical mode: the read buffer could already contain the maximum canon * line of 4096 chars when the mode is switched to non-canonical. * * Locking: n_tty_receive_buf()/producer path: * claims non-exclusive %termios_rwsem * publishes commit_head or canon_head */ static size_t n_tty_receive_buf_common(struct tty_struct *tty, const u8 *cp, const u8 *fp, size_t count, bool flow) { struct n_tty_data *ldata = tty->disc_data; size_t n, rcvd = 0; int room, overflow; down_read(&tty->termios_rwsem); do { /* * When PARMRK is set, each input char may take up to 3 chars * in the read buf; reduce the buffer space avail by 3x * * If we are doing input canonicalization, and there are no * pending newlines, let characters through without limit, so * that erase characters will be handled. Other excess * characters will be beeped. * * paired with store in *_copy_from_read_buf() -- guarantees * the consumer has loaded the data in read_buf up to the new * read_tail (so this producer will not overwrite unread data) */ size_t tail = smp_load_acquire(&ldata->read_tail); room = N_TTY_BUF_SIZE - (ldata->read_head - tail); if (I_PARMRK(tty)) room = DIV_ROUND_UP(room, 3); room--; if (room <= 0) { overflow = ldata->icanon && ldata->canon_head == tail; if (overflow && room < 0) ldata->read_head--; room = overflow; WRITE_ONCE(ldata->no_room, flow && !room); } else overflow = 0; n = min_t(size_t, count, room); if (!n) break; /* ignore parity errors if handling overflow */ if (!overflow || !fp || *fp != TTY_PARITY) __receive_buf(tty, cp, fp, n); cp += n; if (fp) fp += n; count -= n; rcvd += n; } while (!test_bit(TTY_LDISC_CHANGING, &tty->flags)); tty->receive_room = room; /* Unthrottle if handling overflow on pty */ if (tty->driver->type == TTY_DRIVER_TYPE_PTY) { if (overflow) { tty_set_flow_change(tty, TTY_UNTHROTTLE_SAFE); tty_unthrottle_safe(tty); __tty_set_flow_change(tty, 0); } } else n_tty_check_throttle(tty); if (unlikely(ldata->no_room)) { /* * Barrier here is to ensure to read the latest read_tail in * chars_in_buffer() and to make sure that read_tail is not loaded * before ldata->no_room is set. */ smp_mb(); if (!chars_in_buffer(tty)) n_tty_kick_worker(tty); } up_read(&tty->termios_rwsem); return rcvd; } static void n_tty_receive_buf(struct tty_struct *tty, const u8 *cp, const u8 *fp, size_t count) { n_tty_receive_buf_common(tty, cp, fp, count, false); } static size_t n_tty_receive_buf2(struct tty_struct *tty, const u8 *cp, const u8 *fp, size_t count) { return n_tty_receive_buf_common(tty, cp, fp, count, true); } /** * n_tty_set_termios - termios data changed * @tty: terminal * @old: previous data * * Called by the tty layer when the user changes termios flags so that the line * discipline can plan ahead. This function cannot sleep and is protected from * re-entry by the tty layer. The user is guaranteed that this function will * not be re-entered or in progress when the ldisc is closed. * * Locking: Caller holds @tty->termios_rwsem */ static void n_tty_set_termios(struct tty_struct *tty, const struct ktermios *old) { struct n_tty_data *ldata = tty->disc_data; if (!old || (old->c_lflag ^ tty->termios.c_lflag) & (ICANON | EXTPROC)) { bitmap_zero(ldata->read_flags, N_TTY_BUF_SIZE); ldata->line_start = ldata->read_tail; if (!L_ICANON(tty) || !read_cnt(ldata)) { ldata->canon_head = ldata->read_tail; ldata->push = 0; } else { set_bit(MASK(ldata->read_head - 1), ldata->read_flags); ldata->canon_head = ldata->read_head; ldata->push = 1; } ldata->commit_head = ldata->read_head; ldata->erasing = 0; ldata->lnext = 0; } ldata->icanon = (L_ICANON(tty) != 0); if (I_ISTRIP(tty) || I_IUCLC(tty) || I_IGNCR(tty) || I_ICRNL(tty) || I_INLCR(tty) || L_ICANON(tty) || I_IXON(tty) || L_ISIG(tty) || L_ECHO(tty) || I_PARMRK(tty)) { bitmap_zero(ldata->char_map, 256); if (I_IGNCR(tty) || I_ICRNL(tty)) set_bit('\r', ldata->char_map); if (I_INLCR(tty)) set_bit('\n', ldata->char_map); if (L_ICANON(tty)) { set_bit(ERASE_CHAR(tty), ldata->char_map); set_bit(KILL_CHAR(tty), ldata->char_map); set_bit(EOF_CHAR(tty), ldata->char_map); set_bit('\n', ldata->char_map); set_bit(EOL_CHAR(tty), ldata->char_map); if (L_IEXTEN(tty)) { set_bit(WERASE_CHAR(tty), ldata->char_map); set_bit(LNEXT_CHAR(tty), ldata->char_map); set_bit(EOL2_CHAR(tty), ldata->char_map); if (L_ECHO(tty)) set_bit(REPRINT_CHAR(tty), ldata->char_map); } } if (I_IXON(tty)) { set_bit(START_CHAR(tty), ldata->char_map); set_bit(STOP_CHAR(tty), ldata->char_map); } if (L_ISIG(tty)) { set_bit(INTR_CHAR(tty), ldata->char_map); set_bit(QUIT_CHAR(tty), ldata->char_map); set_bit(SUSP_CHAR(tty), ldata->char_map); } clear_bit(__DISABLED_CHAR, ldata->char_map); ldata->raw = 0; ldata->real_raw = 0; } else { ldata->raw = 1; if ((I_IGNBRK(tty) || (!I_BRKINT(tty) && !I_PARMRK(tty))) && (I_IGNPAR(tty) || !I_INPCK(tty)) && (tty->driver->flags & TTY_DRIVER_REAL_RAW)) ldata->real_raw = 1; else ldata->real_raw = 0; } /* * Fix tty hang when I_IXON(tty) is cleared, but the tty * been stopped by STOP_CHAR(tty) before it. */ if (!I_IXON(tty) && old && (old->c_iflag & IXON) && !tty->flow.tco_stopped) { start_tty(tty); process_echoes(tty); } /* The termios change make the tty ready for I/O */ wake_up_interruptible(&tty->write_wait); wake_up_interruptible(&tty->read_wait); } /** * n_tty_close - close the ldisc for this tty * @tty: device * * Called from the terminal layer when this line discipline is being shut down, * either because of a close or becsuse of a discipline change. The function * will not be called while other ldisc methods are in progress. */ static void n_tty_close(struct tty_struct *tty) { struct n_tty_data *ldata = tty->disc_data; if (tty->link) n_tty_packet_mode_flush(tty); down_write(&tty->termios_rwsem); vfree(ldata); tty->disc_data = NULL; up_write(&tty->termios_rwsem); } /** * n_tty_open - open an ldisc * @tty: terminal to open * * Called when this line discipline is being attached to the terminal device. * Can sleep. Called serialized so that no other events will occur in parallel. * No further open will occur until a close. */ static int n_tty_open(struct tty_struct *tty) { struct n_tty_data *ldata; /* Currently a malloc failure here can panic */ ldata = vzalloc(sizeof(*ldata)); if (!ldata) return -ENOMEM; ldata->overrun_time = jiffies; mutex_init(&ldata->atomic_read_lock); mutex_init(&ldata->output_lock); tty->disc_data = ldata; tty->closing = 0; /* indicate buffer work may resume */ clear_bit(TTY_LDISC_HALTED, &tty->flags); n_tty_set_termios(tty, NULL); tty_unthrottle(tty); return 0; } static inline int input_available_p(const struct tty_struct *tty, int poll) { const struct n_tty_data *ldata = tty->disc_data; int amt = poll && !TIME_CHAR(tty) && MIN_CHAR(tty) ? MIN_CHAR(tty) : 1; if (ldata->icanon && !L_EXTPROC(tty)) return ldata->canon_head != ldata->read_tail; else return ldata->commit_head - ldata->read_tail >= amt; } /** * copy_from_read_buf - copy read data directly * @tty: terminal device * @kbp: data * @nr: size of data * * Helper function to speed up n_tty_read(). It is only called when %ICANON is * off; it copies characters straight from the tty queue. * * Returns: true if it successfully copied data, but there is still more data * to be had. * * Locking: * * called under the @ldata->atomic_read_lock sem * * n_tty_read()/consumer path: * caller holds non-exclusive %termios_rwsem; * read_tail published */ static bool copy_from_read_buf(const struct tty_struct *tty, u8 **kbp, size_t *nr) { struct n_tty_data *ldata = tty->disc_data; size_t n; bool is_eof; size_t head = smp_load_acquire(&ldata->commit_head); size_t tail = MASK(ldata->read_tail); n = min3(head - ldata->read_tail, N_TTY_BUF_SIZE - tail, *nr); if (!n) return false; u8 *from = read_buf_addr(ldata, tail); memcpy(*kbp, from, n); is_eof = n == 1 && *from == EOF_CHAR(tty); tty_audit_add_data(tty, from, n); zero_buffer(tty, from, n); smp_store_release(&ldata->read_tail, ldata->read_tail + n); /* Turn single EOF into zero-length read */ if (L_EXTPROC(tty) && ldata->icanon && is_eof && head == ldata->read_tail) return false; *kbp += n; *nr -= n; /* If we have more to copy, let the caller know */ return head != ldata->read_tail; } /** * canon_copy_from_read_buf - copy read data in canonical mode * @tty: terminal device * @kbp: data * @nr: size of data * * Helper function for n_tty_read(). It is only called when %ICANON is on; it * copies one line of input up to and including the line-delimiting character * into the result buffer. * * Note: When termios is changed from non-canonical to canonical mode and the * read buffer contains data, n_tty_set_termios() simulates an EOF push (as if * C-d were input) _without_ the %DISABLED_CHAR in the buffer. This causes data * already processed as input to be immediately available as input although a * newline has not been received. * * Locking: * * called under the %atomic_read_lock mutex * * n_tty_read()/consumer path: * caller holds non-exclusive %termios_rwsem; * read_tail published */ static bool canon_copy_from_read_buf(const struct tty_struct *tty, u8 **kbp, size_t *nr) { struct n_tty_data *ldata = tty->disc_data; size_t n, size, more, c; size_t eol; size_t tail, canon_head; int found = 0; /* N.B. avoid overrun if nr == 0 */ if (!*nr) return false; canon_head = smp_load_acquire(&ldata->canon_head); n = min(*nr, canon_head - ldata->read_tail); tail = MASK(ldata->read_tail); size = min_t(size_t, tail + n, N_TTY_BUF_SIZE); n_tty_trace("%s: nr:%zu tail:%zu n:%zu size:%zu\n", __func__, *nr, tail, n, size); eol = find_next_bit(ldata->read_flags, size, tail); more = n - (size - tail); if (eol == N_TTY_BUF_SIZE && more) { /* scan wrapped without finding set bit */ eol = find_first_bit(ldata->read_flags, more); found = eol != more; } else found = eol != size; n = eol - tail; if (n > N_TTY_BUF_SIZE) n += N_TTY_BUF_SIZE; c = n + found; if (!found || read_buf(ldata, eol) != __DISABLED_CHAR) n = c; n_tty_trace("%s: eol:%zu found:%d n:%zu c:%zu tail:%zu more:%zu\n", __func__, eol, found, n, c, tail, more); tty_copy(tty, *kbp, tail, n); *kbp += n; *nr -= n; if (found) clear_bit(eol, ldata->read_flags); smp_store_release(&ldata->read_tail, ldata->read_tail + c); if (found) { if (!ldata->push) ldata->line_start = ldata->read_tail; else ldata->push = 0; tty_audit_push(); return false; } /* No EOL found - do a continuation retry if there is more data */ return ldata->read_tail != canon_head; } /* * If we finished a read at the exact location of an * EOF (special EOL character that's a __DISABLED_CHAR) * in the stream, silently eat the EOF. */ static void canon_skip_eof(struct n_tty_data *ldata) { size_t tail, canon_head; canon_head = smp_load_acquire(&ldata->canon_head); tail = ldata->read_tail; // No data? if (tail == canon_head) return; // See if the tail position is EOF in the circular buffer tail &= (N_TTY_BUF_SIZE - 1); if (!test_bit(tail, ldata->read_flags)) return; if (read_buf(ldata, tail) != __DISABLED_CHAR) return; // Clear the EOL bit, skip the EOF char. clear_bit(tail, ldata->read_flags); smp_store_release(&ldata->read_tail, ldata->read_tail + 1); } /** * job_control - check job control * @tty: tty * @file: file handle * * Perform job control management checks on this @file/@tty descriptor and if * appropriate send any needed signals and return a negative error code if * action should be taken. * * Locking: * * redirected write test is safe * * current->signal->tty check is safe * * ctrl.lock to safely reference @tty->ctrl.pgrp */ static int job_control(struct tty_struct *tty, struct file *file) { /* Job control check -- must be done at start and after every sleep (POSIX.1 7.1.1.4). */ /* NOTE: not yet done after every sleep pending a thorough check of the logic of this change. -- jlc */ /* don't stop on /dev/console */ if (file->f_op->write_iter == redirected_tty_write) return 0; return __tty_check_change(tty, SIGTTIN); } /** * n_tty_read - read function for tty * @tty: tty device * @file: file object * @kbuf: kernelspace buffer pointer * @nr: size of I/O * @cookie: if non-%NULL, this is a continuation read * @offset: where to continue reading from (unused in n_tty) * * Perform reads for the line discipline. We are guaranteed that the line * discipline will not be closed under us but we may get multiple parallel * readers and must handle this ourselves. We may also get a hangup. Always * called in user context, may sleep. * * This code must be sure never to sleep through a hangup. * * Locking: n_tty_read()/consumer path: * claims non-exclusive termios_rwsem; * publishes read_tail */ static ssize_t n_tty_read(struct tty_struct *tty, struct file *file, u8 *kbuf, size_t nr, void **cookie, unsigned long offset) { struct n_tty_data *ldata = tty->disc_data; u8 *kb = kbuf; DEFINE_WAIT_FUNC(wait, woken_wake_function); int minimum, time; ssize_t retval; long timeout; bool packet; size_t old_tail; /* * Is this a continuation of a read started earler? * * If so, we still hold the atomic_read_lock and the * termios_rwsem, and can just continue to copy data. */ if (*cookie) { if (ldata->icanon && !L_EXTPROC(tty)) { /* * If we have filled the user buffer, see * if we should skip an EOF character before * releasing the lock and returning done. */ if (!nr) canon_skip_eof(ldata); else if (canon_copy_from_read_buf(tty, &kb, &nr)) return kb - kbuf; } else { if (copy_from_read_buf(tty, &kb, &nr)) return kb - kbuf; } /* No more data - release locks and stop retries */ n_tty_kick_worker(tty); n_tty_check_unthrottle(tty); up_read(&tty->termios_rwsem); mutex_unlock(&ldata->atomic_read_lock); *cookie = NULL; return kb - kbuf; } retval = job_control(tty, file); if (retval < 0) return retval; /* * Internal serialization of reads. */ if (file->f_flags & O_NONBLOCK) { if (!mutex_trylock(&ldata->atomic_read_lock)) return -EAGAIN; } else { if (mutex_lock_interruptible(&ldata->atomic_read_lock)) return -ERESTARTSYS; } down_read(&tty->termios_rwsem); minimum = time = 0; timeout = MAX_SCHEDULE_TIMEOUT; if (!ldata->icanon) { minimum = MIN_CHAR(tty); if (minimum) { time = (HZ / 10) * TIME_CHAR(tty); } else { timeout = (HZ / 10) * TIME_CHAR(tty); minimum = 1; } } packet = tty->ctrl.packet; old_tail = ldata->read_tail; add_wait_queue(&tty->read_wait, &wait); while (nr) { /* First test for status change. */ if (packet && tty->link->ctrl.pktstatus) { u8 cs; if (kb != kbuf) break; spin_lock_irq(&tty->link->ctrl.lock); cs = tty->link->ctrl.pktstatus; tty->link->ctrl.pktstatus = 0; spin_unlock_irq(&tty->link->ctrl.lock); *kb++ = cs; nr--; break; } if (!input_available_p(tty, 0)) { up_read(&tty->termios_rwsem); tty_buffer_flush_work(tty->port); down_read(&tty->termios_rwsem); if (!input_available_p(tty, 0)) { if (test_bit(TTY_OTHER_CLOSED, &tty->flags)) { retval = -EIO; break; } if (tty_hung_up_p(file)) break; /* * Abort readers for ttys which never actually * get hung up. See __tty_hangup(). */ if (test_bit(TTY_HUPPING, &tty->flags)) break; if (!timeout) break; if (tty_io_nonblock(tty, file)) { retval = -EAGAIN; break; } if (signal_pending(current)) { retval = -ERESTARTSYS; break; } up_read(&tty->termios_rwsem); timeout = wait_woken(&wait, TASK_INTERRUPTIBLE, timeout); down_read(&tty->termios_rwsem); continue; } } if (ldata->icanon && !L_EXTPROC(tty)) { if (canon_copy_from_read_buf(tty, &kb, &nr)) goto more_to_be_read; } else { /* Deal with packet mode. */ if (packet && kb == kbuf) { *kb++ = TIOCPKT_DATA; nr--; } /* * Copy data, and if there is more to be had * and we have nothing more to wait for, then * let's mark us for retries. * * NOTE! We return here with both the termios_sem * and atomic_read_lock still held, the retries * will release them when done. */ if (copy_from_read_buf(tty, &kb, &nr) && kb - kbuf >= minimum) { more_to_be_read: remove_wait_queue(&tty->read_wait, &wait); *cookie = cookie; return kb - kbuf; } } n_tty_check_unthrottle(tty); if (kb - kbuf >= minimum) break; if (time) timeout = time; } if (old_tail != ldata->read_tail) { /* * Make sure no_room is not read in n_tty_kick_worker() * before setting ldata->read_tail in copy_from_read_buf(). */ smp_mb(); n_tty_kick_worker(tty); } up_read(&tty->termios_rwsem); remove_wait_queue(&tty->read_wait, &wait); mutex_unlock(&ldata->atomic_read_lock); if (kb - kbuf) retval = kb - kbuf; return retval; } /** * n_tty_write - write function for tty * @tty: tty device * @file: file object * @buf: userspace buffer pointer * @nr: size of I/O * * Write function of the terminal device. This is serialized with respect to * other write callers but not to termios changes, reads and other such events. * Since the receive code will echo characters, thus calling driver write * methods, the %output_lock is used in the output processing functions called * here as well as in the echo processing function to protect the column state * and space left in the buffer. * * This code must be sure never to sleep through a hangup. * * Locking: output_lock to protect column state and space left * (note that the process_output*() functions take this lock themselves) */ static ssize_t n_tty_write(struct tty_struct *tty, struct file *file, const u8 *buf, size_t nr) { const u8 *b = buf; DEFINE_WAIT_FUNC(wait, woken_wake_function); ssize_t num, retval = 0; /* Job control check -- must be done at start (POSIX.1 7.1.1.4). */ if (L_TOSTOP(tty) && file->f_op->write_iter != redirected_tty_write) { retval = tty_check_change(tty); if (retval) return retval; } down_read(&tty->termios_rwsem); /* Write out any echoed characters that are still pending */ process_echoes(tty); add_wait_queue(&tty->write_wait, &wait); while (1) { if (signal_pending(current)) { retval = -ERESTARTSYS; break; } if (tty_hung_up_p(file) || (tty->link && !tty->link->count)) { retval = -EIO; break; } if (O_OPOST(tty)) { while (nr > 0) { num = process_output_block(tty, b, nr); if (num < 0) { if (num == -EAGAIN) break; retval = num; goto break_out; } b += num; nr -= num; if (nr == 0) break; if (process_output(*b, tty) < 0) break; b++; nr--; } if (tty->ops->flush_chars) tty->ops->flush_chars(tty); } else { struct n_tty_data *ldata = tty->disc_data; while (nr > 0) { mutex_lock(&ldata->output_lock); num = tty->ops->write(tty, b, nr); mutex_unlock(&ldata->output_lock); if (num < 0) { retval = num; goto break_out; } if (!num) break; b += num; nr -= num; } } if (!nr) break; if (tty_io_nonblock(tty, file)) { retval = -EAGAIN; break; } up_read(&tty->termios_rwsem); wait_woken(&wait, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); down_read(&tty->termios_rwsem); } break_out: remove_wait_queue(&tty->write_wait, &wait); if (nr && tty->fasync) set_bit(TTY_DO_WRITE_WAKEUP, &tty->flags); up_read(&tty->termios_rwsem); return (b - buf) ? b - buf : retval; } /** * n_tty_poll - poll method for N_TTY * @tty: terminal device * @file: file accessing it * @wait: poll table * * Called when the line discipline is asked to poll() for data or for special * events. This code is not serialized with respect to other events save * open/close. * * This code must be sure never to sleep through a hangup. * * Locking: called without the kernel lock held -- fine. */ static __poll_t n_tty_poll(struct tty_struct *tty, struct file *file, poll_table *wait) { __poll_t mask = 0; poll_wait(file, &tty->read_wait, wait); poll_wait(file, &tty->write_wait, wait); if (input_available_p(tty, 1)) mask |= EPOLLIN | EPOLLRDNORM; else { tty_buffer_flush_work(tty->port); if (input_available_p(tty, 1)) mask |= EPOLLIN | EPOLLRDNORM; } if (tty->ctrl.packet && tty->link->ctrl.pktstatus) mask |= EPOLLPRI | EPOLLIN | EPOLLRDNORM; if (test_bit(TTY_OTHER_CLOSED, &tty->flags)) mask |= EPOLLHUP; if (tty_hung_up_p(file)) mask |= EPOLLHUP; if (tty->ops->write && !tty_is_writelocked(tty) && tty_chars_in_buffer(tty) < WAKEUP_CHARS && tty_write_room(tty) > 0) mask |= EPOLLOUT | EPOLLWRNORM; return mask; } static unsigned long inq_canon(struct n_tty_data *ldata) { size_t nr, head, tail; if (ldata->canon_head == ldata->read_tail) return 0; head = ldata->canon_head; tail = ldata->read_tail; nr = head - tail; /* Skip EOF-chars.. */ while (MASK(head) != MASK(tail)) { if (test_bit(MASK(tail), ldata->read_flags) && read_buf(ldata, tail) == __DISABLED_CHAR) nr--; tail++; } return nr; } static int n_tty_ioctl(struct tty_struct *tty, unsigned int cmd, unsigned long arg) { struct n_tty_data *ldata = tty->disc_data; unsigned int num; switch (cmd) { case TIOCOUTQ: return put_user(tty_chars_in_buffer(tty), (int __user *) arg); case TIOCINQ: down_write(&tty->termios_rwsem); if (L_ICANON(tty) && !L_EXTPROC(tty)) num = inq_canon(ldata); else num = read_cnt(ldata); up_write(&tty->termios_rwsem); return put_user(num, (unsigned int __user *) arg); default: return n_tty_ioctl_helper(tty, cmd, arg); } } static struct tty_ldisc_ops n_tty_ops = { .owner = THIS_MODULE, .num = N_TTY, .name = "n_tty", .open = n_tty_open, .close = n_tty_close, .flush_buffer = n_tty_flush_buffer, .read = n_tty_read, .write = n_tty_write, .ioctl = n_tty_ioctl, .set_termios = n_tty_set_termios, .poll = n_tty_poll, .receive_buf = n_tty_receive_buf, .write_wakeup = n_tty_write_wakeup, .receive_buf2 = n_tty_receive_buf2, .lookahead_buf = n_tty_lookahead_flow_ctrl, }; /** * n_tty_inherit_ops - inherit N_TTY methods * @ops: struct tty_ldisc_ops where to save N_TTY methods * * Enables a 'subclass' line discipline to 'inherit' N_TTY methods. */ void n_tty_inherit_ops(struct tty_ldisc_ops *ops) { *ops = n_tty_ops; ops->owner = NULL; } EXPORT_SYMBOL_GPL(n_tty_inherit_ops); void __init n_tty_init(void) { tty_register_ldisc(&n_tty_ops); } |
| 1 1 2 1 1 6 2 5 1 4 2 2 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (c) 2008-2009 Atheros Communications Inc. */ #include <linux/module.h> #include <linux/kernel.h> #include <linux/init.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/errno.h> #include <linux/firmware.h> #include <linux/usb.h> #include <linux/unaligned.h> #include <net/bluetooth/bluetooth.h> #define VERSION "1.0" #define ATH3K_FIRMWARE "ath3k-1.fw" #define ATH3K_DNLOAD 0x01 #define ATH3K_GETSTATE 0x05 #define ATH3K_SET_NORMAL_MODE 0x07 #define ATH3K_GETVERSION 0x09 #define USB_REG_SWITCH_VID_PID 0x0a #define ATH3K_MODE_MASK 0x3F #define ATH3K_NORMAL_MODE 0x0E #define ATH3K_PATCH_UPDATE 0x80 #define ATH3K_SYSCFG_UPDATE 0x40 #define ATH3K_XTAL_FREQ_26M 0x00 #define ATH3K_XTAL_FREQ_40M 0x01 #define ATH3K_XTAL_FREQ_19P2 0x02 #define ATH3K_NAME_LEN 0xFF struct ath3k_version { __le32 rom_version; __le32 build_version; __le32 ram_version; __u8 ref_clock; __u8 reserved[7]; } __packed; static const struct usb_device_id ath3k_table[] = { /* Atheros AR3011 */ { USB_DEVICE(0x0CF3, 0x3000) }, /* Atheros AR3011 with sflash firmware*/ { USB_DEVICE(0x0489, 0xE027) }, { USB_DEVICE(0x0489, 0xE03D) }, { USB_DEVICE(0x04F2, 0xAFF1) }, { USB_DEVICE(0x0930, 0x0215) }, { USB_DEVICE(0x0CF3, 0x3002) }, { USB_DEVICE(0x0CF3, 0xE019) }, { USB_DEVICE(0x13d3, 0x3304) }, /* Atheros AR9285 Malbec with sflash firmware */ { USB_DEVICE(0x03F0, 0x311D) }, /* Atheros AR3012 with sflash firmware*/ { USB_DEVICE(0x0489, 0xe04d) }, { USB_DEVICE(0x0489, 0xe04e) }, { USB_DEVICE(0x0489, 0xe057) }, { USB_DEVICE(0x0489, 0xe056) }, { USB_DEVICE(0x0489, 0xe05f) }, { USB_DEVICE(0x0489, 0xe076) }, { USB_DEVICE(0x0489, 0xe078) }, { USB_DEVICE(0x0489, 0xe095) }, { USB_DEVICE(0x04c5, 0x1330) }, { USB_DEVICE(0x04CA, 0x3004) }, { USB_DEVICE(0x04CA, 0x3005) }, { USB_DEVICE(0x04CA, 0x3006) }, { USB_DEVICE(0x04CA, 0x3007) }, { USB_DEVICE(0x04CA, 0x3008) }, { USB_DEVICE(0x04CA, 0x300b) }, { USB_DEVICE(0x04CA, 0x300d) }, { USB_DEVICE(0x04CA, 0x300f) }, { USB_DEVICE(0x04CA, 0x3010) }, { USB_DEVICE(0x04CA, 0x3014) }, { USB_DEVICE(0x04CA, 0x3018) }, { USB_DEVICE(0x0930, 0x0219) }, { USB_DEVICE(0x0930, 0x021c) }, { USB_DEVICE(0x0930, 0x0220) }, { USB_DEVICE(0x0930, 0x0227) }, { USB_DEVICE(0x0b05, 0x17d0) }, { USB_DEVICE(0x0CF3, 0x0036) }, { USB_DEVICE(0x0CF3, 0x3004) }, { USB_DEVICE(0x0CF3, 0x3008) }, { USB_DEVICE(0x0CF3, 0x311D) }, { USB_DEVICE(0x0CF3, 0x311E) }, { USB_DEVICE(0x0CF3, 0x311F) }, { USB_DEVICE(0x0cf3, 0x3121) }, { USB_DEVICE(0x0CF3, 0x817a) }, { USB_DEVICE(0x0CF3, 0x817b) }, { USB_DEVICE(0x0cf3, 0xe003) }, { USB_DEVICE(0x0CF3, 0xE004) }, { USB_DEVICE(0x0CF3, 0xE005) }, { USB_DEVICE(0x0CF3, 0xE006) }, { USB_DEVICE(0x13d3, 0x3362) }, { USB_DEVICE(0x13d3, 0x3375) }, { USB_DEVICE(0x13d3, 0x3393) }, { USB_DEVICE(0x13d3, 0x3395) }, { USB_DEVICE(0x13d3, 0x3402) }, { USB_DEVICE(0x13d3, 0x3408) }, { USB_DEVICE(0x13d3, 0x3423) }, { USB_DEVICE(0x13d3, 0x3432) }, { USB_DEVICE(0x13d3, 0x3472) }, { USB_DEVICE(0x13d3, 0x3474) }, { USB_DEVICE(0x13d3, 0x3487) }, { USB_DEVICE(0x13d3, 0x3490) }, /* Atheros AR5BBU12 with sflash firmware */ { USB_DEVICE(0x0489, 0xE02C) }, /* Atheros AR5BBU22 with sflash firmware */ { USB_DEVICE(0x0489, 0xE036) }, { USB_DEVICE(0x0489, 0xE03C) }, { } /* Terminating entry */ }; MODULE_DEVICE_TABLE(usb, ath3k_table); #define BTUSB_ATH3012 0x80 /* This table is to load patch and sysconfig files * for AR3012 */ static const struct usb_device_id ath3k_blist_tbl[] = { /* Atheros AR3012 with sflash firmware*/ { USB_DEVICE(0x0489, 0xe04e), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xe04d), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xe056), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xe057), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xe05f), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xe076), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xe078), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xe095), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04c5, 0x1330), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3004), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3005), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3006), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3007), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3008), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x300b), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x300d), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x300f), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3010), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3014), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x04ca, 0x3018), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0930, 0x0219), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0930, 0x021c), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0930, 0x0220), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0930, 0x0227), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0b05, 0x17d0), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0CF3, 0x0036), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0cf3, 0x3004), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0cf3, 0x3008), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0cf3, 0x311D), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0cf3, 0x311E), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0cf3, 0x311F), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0cf3, 0x3121), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0CF3, 0x817a), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0CF3, 0x817b), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0cf3, 0xe004), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0cf3, 0xe005), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0cf3, 0xe006), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0cf3, 0xe003), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3362), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3375), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3393), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3395), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3402), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3408), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3423), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3432), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3472), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3474), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3487), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x13d3, 0x3490), .driver_info = BTUSB_ATH3012 }, /* Atheros AR5BBU22 with sflash firmware */ { USB_DEVICE(0x0489, 0xE036), .driver_info = BTUSB_ATH3012 }, { USB_DEVICE(0x0489, 0xE03C), .driver_info = BTUSB_ATH3012 }, { } /* Terminating entry */ }; static inline void ath3k_log_failed_loading(int err, int len, int size, int count) { BT_ERR("Firmware loading err = %d, len = %d, size = %d, count = %d", err, len, size, count); } #define USB_REQ_DFU_DNLOAD 1 #define BULK_SIZE 4096 #define FW_HDR_SIZE 20 #define TIMEGAP_USEC_MIN 50 #define TIMEGAP_USEC_MAX 100 static int ath3k_load_firmware(struct usb_device *udev, const struct firmware *firmware) { u8 *send_buf; int len = 0; int err, pipe, size, sent = 0; int count = firmware->size; BT_DBG("udev %p", udev); send_buf = kmalloc(BULK_SIZE, GFP_KERNEL); if (!send_buf) { BT_ERR("Can't allocate memory chunk for firmware"); return -ENOMEM; } err = usb_control_msg_send(udev, 0, USB_REQ_DFU_DNLOAD, USB_TYPE_VENDOR, 0, 0, firmware->data, FW_HDR_SIZE, USB_CTRL_SET_TIMEOUT, GFP_KERNEL); if (err) { BT_ERR("Can't change to loading configuration err"); goto error; } sent += FW_HDR_SIZE; count -= FW_HDR_SIZE; pipe = usb_sndbulkpipe(udev, 0x02); while (count) { /* workaround the compatibility issue with xHCI controller*/ usleep_range(TIMEGAP_USEC_MIN, TIMEGAP_USEC_MAX); size = min_t(uint, count, BULK_SIZE); memcpy(send_buf, firmware->data + sent, size); err = usb_bulk_msg(udev, pipe, send_buf, size, &len, 3000); if (err || len != size) { ath3k_log_failed_loading(err, len, size, count); goto error; } sent += size; count -= size; } error: kfree(send_buf); return err; } static int ath3k_get_state(struct usb_device *udev, unsigned char *state) { return usb_control_msg_recv(udev, 0, ATH3K_GETSTATE, USB_TYPE_VENDOR | USB_DIR_IN, 0, 0, state, 1, USB_CTRL_SET_TIMEOUT, GFP_KERNEL); } static int ath3k_get_version(struct usb_device *udev, struct ath3k_version *version) { return usb_control_msg_recv(udev, 0, ATH3K_GETVERSION, USB_TYPE_VENDOR | USB_DIR_IN, 0, 0, version, sizeof(*version), USB_CTRL_SET_TIMEOUT, GFP_KERNEL); } static int ath3k_load_fwfile(struct usb_device *udev, const struct firmware *firmware) { u8 *send_buf; int len = 0; int err, pipe, size, count, sent = 0; int ret; count = firmware->size; send_buf = kmalloc(BULK_SIZE, GFP_KERNEL); if (!send_buf) { BT_ERR("Can't allocate memory chunk for firmware"); return -ENOMEM; } size = min_t(uint, count, FW_HDR_SIZE); ret = usb_control_msg_send(udev, 0, ATH3K_DNLOAD, USB_TYPE_VENDOR, 0, 0, firmware->data, size, USB_CTRL_SET_TIMEOUT, GFP_KERNEL); if (ret) { BT_ERR("Can't change to loading configuration err"); kfree(send_buf); return ret; } sent += size; count -= size; pipe = usb_sndbulkpipe(udev, 0x02); while (count) { /* workaround the compatibility issue with xHCI controller*/ usleep_range(TIMEGAP_USEC_MIN, TIMEGAP_USEC_MAX); size = min_t(uint, count, BULK_SIZE); memcpy(send_buf, firmware->data + sent, size); err = usb_bulk_msg(udev, pipe, send_buf, size, &len, 3000); if (err || len != size) { ath3k_log_failed_loading(err, len, size, count); kfree(send_buf); return err; } sent += size; count -= size; } kfree(send_buf); return 0; } static void ath3k_switch_pid(struct usb_device *udev) { usb_control_msg_send(udev, 0, USB_REG_SWITCH_VID_PID, USB_TYPE_VENDOR, 0, 0, NULL, 0, USB_CTRL_SET_TIMEOUT, GFP_KERNEL); } static int ath3k_set_normal_mode(struct usb_device *udev) { unsigned char fw_state; int ret; ret = ath3k_get_state(udev, &fw_state); if (ret) { BT_ERR("Can't get state to change to normal mode err"); return ret; } if ((fw_state & ATH3K_MODE_MASK) == ATH3K_NORMAL_MODE) { BT_DBG("firmware was already in normal mode"); return 0; } return usb_control_msg_send(udev, 0, ATH3K_SET_NORMAL_MODE, USB_TYPE_VENDOR, 0, 0, NULL, 0, USB_CTRL_SET_TIMEOUT, GFP_KERNEL); } static int ath3k_load_patch(struct usb_device *udev) { unsigned char fw_state; char filename[ATH3K_NAME_LEN]; const struct firmware *firmware; struct ath3k_version fw_version; __u32 pt_rom_version, pt_build_version; int ret; ret = ath3k_get_state(udev, &fw_state); if (ret) { BT_ERR("Can't get state to change to load ram patch err"); return ret; } if (fw_state & ATH3K_PATCH_UPDATE) { BT_DBG("Patch was already downloaded"); return 0; } ret = ath3k_get_version(udev, &fw_version); if (ret) { BT_ERR("Can't get version to change to load ram patch err"); return ret; } snprintf(filename, ATH3K_NAME_LEN, "ar3k/AthrBT_0x%08x.dfu", le32_to_cpu(fw_version.rom_version)); ret = request_firmware(&firmware, filename, &udev->dev); if (ret < 0) { BT_ERR("Patch file not found %s", filename); return ret; } pt_rom_version = get_unaligned_le32(firmware->data + firmware->size - 8); pt_build_version = get_unaligned_le32(firmware->data + firmware->size - 4); if (pt_rom_version != le32_to_cpu(fw_version.rom_version) || pt_build_version <= le32_to_cpu(fw_version.build_version)) { BT_ERR("Patch file version did not match with firmware"); release_firmware(firmware); return -EINVAL; } ret = ath3k_load_fwfile(udev, firmware); release_firmware(firmware); return ret; } static int ath3k_load_syscfg(struct usb_device *udev) { unsigned char fw_state; char filename[ATH3K_NAME_LEN]; const struct firmware *firmware; struct ath3k_version fw_version; int clk_value, ret; ret = ath3k_get_state(udev, &fw_state); if (ret) { BT_ERR("Can't get state to change to load configuration err"); return -EBUSY; } ret = ath3k_get_version(udev, &fw_version); if (ret) { BT_ERR("Can't get version to change to load ram patch err"); return ret; } switch (fw_version.ref_clock) { case ATH3K_XTAL_FREQ_26M: clk_value = 26; break; case ATH3K_XTAL_FREQ_40M: clk_value = 40; break; case ATH3K_XTAL_FREQ_19P2: clk_value = 19; break; default: clk_value = 0; break; } snprintf(filename, ATH3K_NAME_LEN, "ar3k/ramps_0x%08x_%d%s", le32_to_cpu(fw_version.rom_version), clk_value, ".dfu"); ret = request_firmware(&firmware, filename, &udev->dev); if (ret < 0) { BT_ERR("Configuration file not found %s", filename); return ret; } ret = ath3k_load_fwfile(udev, firmware); release_firmware(firmware); return ret; } static int ath3k_probe(struct usb_interface *intf, const struct usb_device_id *id) { const struct firmware *firmware; struct usb_device *udev = interface_to_usbdev(intf); int ret; BT_DBG("intf %p id %p", intf, id); if (intf->cur_altsetting->desc.bInterfaceNumber != 0) return -ENODEV; /* match device ID in ath3k blacklist table */ if (!id->driver_info) { const struct usb_device_id *match; match = usb_match_id(intf, ath3k_blist_tbl); if (match) id = match; } /* load patch and sysconfig files for AR3012 */ if (id->driver_info & BTUSB_ATH3012) { /* New firmware with patch and sysconfig files already loaded */ if (le16_to_cpu(udev->descriptor.bcdDevice) > 0x0001) return -ENODEV; ret = ath3k_load_patch(udev); if (ret < 0) { BT_ERR("Loading patch file failed"); return ret; } ret = ath3k_load_syscfg(udev); if (ret < 0) { BT_ERR("Loading sysconfig file failed"); return ret; } ret = ath3k_set_normal_mode(udev); if (ret) { BT_ERR("Set normal mode failed"); return ret; } ath3k_switch_pid(udev); return 0; } ret = request_firmware(&firmware, ATH3K_FIRMWARE, &udev->dev); if (ret < 0) { if (ret == -ENOENT) BT_ERR("Firmware file \"%s\" not found", ATH3K_FIRMWARE); else BT_ERR("Firmware file \"%s\" request failed (err=%d)", ATH3K_FIRMWARE, ret); return ret; } ret = ath3k_load_firmware(udev, firmware); release_firmware(firmware); return ret; } static void ath3k_disconnect(struct usb_interface *intf) { BT_DBG("%s intf %p", __func__, intf); } static struct usb_driver ath3k_driver = { .name = "ath3k", .probe = ath3k_probe, .disconnect = ath3k_disconnect, .id_table = ath3k_table, .disable_hub_initiated_lpm = 1, }; module_usb_driver(ath3k_driver); MODULE_AUTHOR("Atheros Communications"); MODULE_DESCRIPTION("Atheros AR30xx firmware driver"); MODULE_VERSION(VERSION); MODULE_LICENSE("GPL"); MODULE_FIRMWARE(ATH3K_FIRMWARE); |
| 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Force feedback support for Logitech RumblePad and Rumblepad 2 * * Copyright (c) 2008 Anssi Hannula <anssi.hannula@gmail.com> */ /* */ #include <linux/input.h> #include <linux/slab.h> #include <linux/hid.h> #include "hid-lg.h" struct lg2ff_device { struct hid_report *report; }; static int play_effect(struct input_dev *dev, void *data, struct ff_effect *effect) { struct hid_device *hid = input_get_drvdata(dev); struct lg2ff_device *lg2ff = data; int weak, strong; strong = effect->u.rumble.strong_magnitude; weak = effect->u.rumble.weak_magnitude; if (weak || strong) { weak = weak * 0xff / 0xffff; strong = strong * 0xff / 0xffff; lg2ff->report->field[0]->value[0] = 0x51; lg2ff->report->field[0]->value[2] = weak; lg2ff->report->field[0]->value[4] = strong; } else { lg2ff->report->field[0]->value[0] = 0xf3; lg2ff->report->field[0]->value[2] = 0x00; lg2ff->report->field[0]->value[4] = 0x00; } hid_hw_request(hid, lg2ff->report, HID_REQ_SET_REPORT); return 0; } int lg2ff_init(struct hid_device *hid) { struct lg2ff_device *lg2ff; struct hid_report *report; struct hid_input *hidinput; struct input_dev *dev; int error; if (list_empty(&hid->inputs)) { hid_err(hid, "no inputs found\n"); return -ENODEV; } hidinput = list_entry(hid->inputs.next, struct hid_input, list); dev = hidinput->input; /* Check that the report looks ok */ report = hid_validate_values(hid, HID_OUTPUT_REPORT, 0, 0, 7); if (!report) return -ENODEV; lg2ff = kmalloc(sizeof(struct lg2ff_device), GFP_KERNEL); if (!lg2ff) return -ENOMEM; set_bit(FF_RUMBLE, dev->ffbit); error = input_ff_create_memless(dev, lg2ff, play_effect); if (error) { kfree(lg2ff); return error; } lg2ff->report = report; report->field[0]->value[0] = 0xf3; report->field[0]->value[1] = 0x00; report->field[0]->value[2] = 0x00; report->field[0]->value[3] = 0x00; report->field[0]->value[4] = 0x00; report->field[0]->value[5] = 0x00; report->field[0]->value[6] = 0x00; hid_hw_request(hid, report, HID_REQ_SET_REPORT); hid_info(hid, "Force feedback for Logitech variant 2 rumble devices by Anssi Hannula <anssi.hannula@gmail.com>\n"); return 0; } |
| 8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | /* * net/tipc/bcast.h: Include file for TIPC broadcast code * * Copyright (c) 2003-2006, 2014-2015, Ericsson AB * Copyright (c) 2005, 2010-2011, Wind River Systems * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the names of the copyright holders nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * Alternatively, this software may be distributed under the terms of the * GNU General Public License ("GPL") version 2 as published by the Free * Software Foundation. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #ifndef _TIPC_BCAST_H #define _TIPC_BCAST_H #include "core.h" struct tipc_node; struct tipc_msg; struct tipc_nl_msg; struct tipc_nlist; struct tipc_nitem; extern const char tipc_bclink_name[]; extern unsigned long sysctl_tipc_bc_retruni; #define TIPC_METHOD_EXPIRE msecs_to_jiffies(5000) #define BCLINK_MODE_BCAST 0x1 #define BCLINK_MODE_RCAST 0x2 #define BCLINK_MODE_SEL 0x4 struct tipc_nlist { struct list_head list; u32 self; u16 remote; bool local; }; void tipc_nlist_init(struct tipc_nlist *nl, u32 self); void tipc_nlist_purge(struct tipc_nlist *nl); void tipc_nlist_add(struct tipc_nlist *nl, u32 node); void tipc_nlist_del(struct tipc_nlist *nl, u32 node); /* Cookie to be used between socket and broadcast layer * @rcast: replicast (instead of broadcast) was used at previous xmit * @mandatory: broadcast/replicast indication was set by user * @deferredq: defer queue to make message in order * @expires: re-evaluate non-mandatory transmit method if we are past this */ struct tipc_mc_method { bool rcast; bool mandatory; struct sk_buff_head deferredq; unsigned long expires; }; int tipc_bcast_init(struct net *net); void tipc_bcast_stop(struct net *net); void tipc_bcast_add_peer(struct net *net, struct tipc_link *l, struct sk_buff_head *xmitq); void tipc_bcast_remove_peer(struct net *net, struct tipc_link *rcv_bcl); void tipc_bcast_inc_bearer_dst_cnt(struct net *net, int bearer_id); void tipc_bcast_dec_bearer_dst_cnt(struct net *net, int bearer_id); int tipc_bcast_get_mtu(struct net *net); void tipc_bcast_toggle_rcast(struct net *net, bool supp); int tipc_mcast_xmit(struct net *net, struct sk_buff_head *pkts, struct tipc_mc_method *method, struct tipc_nlist *dests, u16 *cong_link_cnt); int tipc_bcast_xmit(struct net *net, struct sk_buff_head *pkts, u16 *cong_link_cnt); int tipc_bcast_rcv(struct net *net, struct tipc_link *l, struct sk_buff *skb); void tipc_bcast_ack_rcv(struct net *net, struct tipc_link *l, struct tipc_msg *hdr); int tipc_bcast_sync_rcv(struct net *net, struct tipc_link *l, struct tipc_msg *hdr, struct sk_buff_head *retrq); int tipc_nl_add_bc_link(struct net *net, struct tipc_nl_msg *msg, struct tipc_link *bcl); int tipc_nl_bc_link_set(struct net *net, struct nlattr *attrs[]); int tipc_bclink_reset_stats(struct net *net, struct tipc_link *l); u32 tipc_bcast_get_mode(struct net *net); u32 tipc_bcast_get_broadcast_ratio(struct net *net); void tipc_mcast_filter_msg(struct net *net, struct sk_buff_head *defq, struct sk_buff_head *inputq); static inline void tipc_bcast_lock(struct net *net) { spin_lock_bh(&tipc_net(net)->bclock); } static inline void tipc_bcast_unlock(struct net *net) { spin_unlock_bh(&tipc_net(net)->bclock); } static inline struct tipc_link *tipc_bc_sndlink(struct net *net) { return tipc_net(net)->bcl; } #endif |
| 1123 1120 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 | // SPDX-License-Identifier: GPL-2.0-or-later /* Paravirtualization interfaces Copyright (C) 2006 Rusty Russell IBM Corporation 2007 - x86_64 support added by Glauber de Oliveira Costa, Red Hat Inc */ #include <linux/errno.h> #include <linux/init.h> #include <linux/export.h> #include <linux/efi.h> #include <linux/bcd.h> #include <linux/highmem.h> #include <linux/kprobes.h> #include <linux/pgtable.h> #include <linux/static_call.h> #include <asm/bug.h> #include <asm/paravirt.h> #include <asm/debugreg.h> #include <asm/desc.h> #include <asm/setup.h> #include <asm/time.h> #include <asm/pgalloc.h> #include <asm/irq.h> #include <asm/delay.h> #include <asm/fixmap.h> #include <asm/apic.h> #include <asm/tlbflush.h> #include <asm/timer.h> #include <asm/special_insns.h> #include <asm/tlb.h> #include <asm/io_bitmap.h> #include <asm/gsseg.h> /* stub always returning 0. */ DEFINE_ASM_FUNC(paravirt_ret0, "xor %eax,%eax", .entry.text); void __init default_banner(void) { printk(KERN_INFO "Booting paravirtualized kernel on %s\n", pv_info.name); } #ifdef CONFIG_PARAVIRT_XXL DEFINE_ASM_FUNC(_paravirt_ident_64, "mov %rdi, %rax", .text); DEFINE_ASM_FUNC(pv_native_save_fl, "pushf; pop %rax", .noinstr.text); DEFINE_ASM_FUNC(pv_native_irq_disable, "cli", .noinstr.text); DEFINE_ASM_FUNC(pv_native_irq_enable, "sti", .noinstr.text); DEFINE_ASM_FUNC(pv_native_read_cr2, "mov %cr2, %rax", .noinstr.text); #endif DEFINE_STATIC_KEY_FALSE(virt_spin_lock_key); void __init native_pv_lock_init(void) { if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) static_branch_enable(&virt_spin_lock_key); } static void native_tlb_remove_table(struct mmu_gather *tlb, void *table) { tlb_remove_page(tlb, table); } struct static_key paravirt_steal_enabled; struct static_key paravirt_steal_rq_enabled; static u64 native_steal_clock(int cpu) { return 0; } DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); DEFINE_STATIC_CALL(pv_sched_clock, native_sched_clock); void paravirt_set_sched_clock(u64 (*func)(void)) { static_call_update(pv_sched_clock, func); } /* These are in entry.S */ static struct resource reserve_ioports = { .start = 0, .end = IO_SPACE_LIMIT, .name = "paravirt-ioport", .flags = IORESOURCE_IO | IORESOURCE_BUSY, }; /* * Reserve the whole legacy IO space to prevent any legacy drivers * from wasting time probing for their hardware. This is a fairly * brute-force approach to disabling all non-virtual drivers. * * Note that this must be called very early to have any effect. */ int paravirt_disable_iospace(void) { return request_resource(&ioport_resource, &reserve_ioports); } #ifdef CONFIG_PARAVIRT_XXL static noinstr void pv_native_write_cr2(unsigned long val) { native_write_cr2(val); } static noinstr unsigned long pv_native_get_debugreg(int regno) { return native_get_debugreg(regno); } static noinstr void pv_native_set_debugreg(int regno, unsigned long val) { native_set_debugreg(regno, val); } noinstr void pv_native_wbinvd(void) { native_wbinvd(); } static noinstr void pv_native_safe_halt(void) { native_safe_halt(); } #endif struct pv_info pv_info = { .name = "bare hardware", #ifdef CONFIG_PARAVIRT_XXL .extra_user_64bit_cs = __USER_CS, #endif }; /* 64-bit pagetable entries */ #define PTE_IDENT __PV_IS_CALLEE_SAVE(_paravirt_ident_64) struct paravirt_patch_template pv_ops = { /* Cpu ops. */ .cpu.io_delay = native_io_delay, #ifdef CONFIG_PARAVIRT_XXL .cpu.cpuid = native_cpuid, .cpu.get_debugreg = pv_native_get_debugreg, .cpu.set_debugreg = pv_native_set_debugreg, .cpu.read_cr0 = native_read_cr0, .cpu.write_cr0 = native_write_cr0, .cpu.write_cr4 = native_write_cr4, .cpu.wbinvd = pv_native_wbinvd, .cpu.read_msr = native_read_msr, .cpu.write_msr = native_write_msr, .cpu.read_msr_safe = native_read_msr_safe, .cpu.write_msr_safe = native_write_msr_safe, .cpu.read_pmc = native_read_pmc, .cpu.load_tr_desc = native_load_tr_desc, .cpu.set_ldt = native_set_ldt, .cpu.load_gdt = native_load_gdt, .cpu.load_idt = native_load_idt, .cpu.store_tr = native_store_tr, .cpu.load_tls = native_load_tls, .cpu.load_gs_index = native_load_gs_index, .cpu.write_ldt_entry = native_write_ldt_entry, .cpu.write_gdt_entry = native_write_gdt_entry, .cpu.write_idt_entry = native_write_idt_entry, .cpu.alloc_ldt = paravirt_nop, .cpu.free_ldt = paravirt_nop, .cpu.load_sp0 = native_load_sp0, #ifdef CONFIG_X86_IOPL_IOPERM .cpu.invalidate_io_bitmap = native_tss_invalidate_io_bitmap, .cpu.update_io_bitmap = native_tss_update_io_bitmap, #endif .cpu.start_context_switch = paravirt_nop, .cpu.end_context_switch = paravirt_nop, /* Irq ops. */ .irq.save_fl = __PV_IS_CALLEE_SAVE(pv_native_save_fl), .irq.irq_disable = __PV_IS_CALLEE_SAVE(pv_native_irq_disable), .irq.irq_enable = __PV_IS_CALLEE_SAVE(pv_native_irq_enable), .irq.safe_halt = pv_native_safe_halt, .irq.halt = native_halt, #endif /* CONFIG_PARAVIRT_XXL */ /* Mmu ops. */ .mmu.flush_tlb_user = native_flush_tlb_local, .mmu.flush_tlb_kernel = native_flush_tlb_global, .mmu.flush_tlb_one_user = native_flush_tlb_one_user, .mmu.flush_tlb_multi = native_flush_tlb_multi, .mmu.tlb_remove_table = native_tlb_remove_table, .mmu.exit_mmap = paravirt_nop, .mmu.notify_page_enc_status_changed = paravirt_nop, #ifdef CONFIG_PARAVIRT_XXL .mmu.read_cr2 = __PV_IS_CALLEE_SAVE(pv_native_read_cr2), .mmu.write_cr2 = pv_native_write_cr2, .mmu.read_cr3 = __native_read_cr3, .mmu.write_cr3 = native_write_cr3, .mmu.pgd_alloc = __paravirt_pgd_alloc, .mmu.pgd_free = paravirt_nop, .mmu.alloc_pte = paravirt_nop, .mmu.alloc_pmd = paravirt_nop, .mmu.alloc_pud = paravirt_nop, .mmu.alloc_p4d = paravirt_nop, .mmu.release_pte = paravirt_nop, .mmu.release_pmd = paravirt_nop, .mmu.release_pud = paravirt_nop, .mmu.release_p4d = paravirt_nop, .mmu.set_pte = native_set_pte, .mmu.set_pmd = native_set_pmd, .mmu.ptep_modify_prot_start = __ptep_modify_prot_start, .mmu.ptep_modify_prot_commit = __ptep_modify_prot_commit, .mmu.set_pud = native_set_pud, .mmu.pmd_val = PTE_IDENT, .mmu.make_pmd = PTE_IDENT, .mmu.pud_val = PTE_IDENT, .mmu.make_pud = PTE_IDENT, .mmu.set_p4d = native_set_p4d, #if CONFIG_PGTABLE_LEVELS >= 5 .mmu.p4d_val = PTE_IDENT, .mmu.make_p4d = PTE_IDENT, .mmu.set_pgd = native_set_pgd, #endif /* CONFIG_PGTABLE_LEVELS >= 5 */ .mmu.pte_val = PTE_IDENT, .mmu.pgd_val = PTE_IDENT, .mmu.make_pte = PTE_IDENT, .mmu.make_pgd = PTE_IDENT, .mmu.enter_mmap = paravirt_nop, .mmu.lazy_mode = { .enter = paravirt_nop, .leave = paravirt_nop, .flush = paravirt_nop, }, .mmu.set_fixmap = native_set_fixmap, #endif /* CONFIG_PARAVIRT_XXL */ #if defined(CONFIG_PARAVIRT_SPINLOCKS) /* Lock ops. */ #ifdef CONFIG_SMP .lock.queued_spin_lock_slowpath = native_queued_spin_lock_slowpath, .lock.queued_spin_unlock = PV_CALLEE_SAVE(__native_queued_spin_unlock), .lock.wait = paravirt_nop, .lock.kick = paravirt_nop, .lock.vcpu_is_preempted = PV_CALLEE_SAVE(__native_vcpu_is_preempted), #endif /* SMP */ #endif }; #ifdef CONFIG_PARAVIRT_XXL NOKPROBE_SYMBOL(native_load_idt); #endif EXPORT_SYMBOL(pv_ops); EXPORT_SYMBOL_GPL(pv_info); |
| 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 16 17 17 17 1 1 17 1 1 16 17 17 17 16 1 16 11 1 11 1 16 1 1 1 1 16 16 5 6 5 11 11 11 10 15 15 15 1 1 21 21 15 15 15 15 8 17 12 8 15 15 15 15 15 15 15 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 | // SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2000-2003 Silicon Graphics, Inc. * All Rights Reserved. */ #include "xfs.h" #include "xfs_fs.h" #include "xfs_format.h" #include "xfs_log_format.h" #include "xfs_shared.h" #include "xfs_trans_resv.h" #include "xfs_bit.h" #include "xfs_mount.h" #include "xfs_defer.h" #include "xfs_inode.h" #include "xfs_bmap.h" #include "xfs_quota.h" #include "xfs_trans.h" #include "xfs_buf_item.h" #include "xfs_trans_space.h" #include "xfs_trans_priv.h" #include "xfs_qm.h" #include "xfs_trace.h" #include "xfs_log.h" #include "xfs_bmap_btree.h" #include "xfs_error.h" #include "xfs_health.h" /* * Lock order: * * ip->i_lock * qi->qi_tree_lock * dquot->q_qlock (xfs_dqlock() and friends) * dquot->q_flush (xfs_dqflock() and friends) * qi->qi_lru_lock * * If two dquots need to be locked the order is user before group/project, * otherwise by the lowest id first, see xfs_dqlock2. */ struct kmem_cache *xfs_dqtrx_cache; static struct kmem_cache *xfs_dquot_cache; static struct lock_class_key xfs_dquot_group_class; static struct lock_class_key xfs_dquot_project_class; /* Record observations of quota corruption with the health tracking system. */ static void xfs_dquot_mark_sick( struct xfs_dquot *dqp) { struct xfs_mount *mp = dqp->q_mount; switch (dqp->q_type) { case XFS_DQTYPE_USER: xfs_fs_mark_sick(mp, XFS_SICK_FS_UQUOTA); break; case XFS_DQTYPE_GROUP: xfs_fs_mark_sick(mp, XFS_SICK_FS_GQUOTA); break; case XFS_DQTYPE_PROJ: xfs_fs_mark_sick(mp, XFS_SICK_FS_PQUOTA); break; default: ASSERT(0); break; } } /* * Detach the dquot buffer if it's still attached, because we can get called * through dqpurge after a log shutdown. Caller must hold the dqflock or have * otherwise isolated the dquot. */ void xfs_dquot_detach_buf( struct xfs_dquot *dqp) { struct xfs_dq_logitem *qlip = &dqp->q_logitem; struct xfs_buf *bp = NULL; spin_lock(&qlip->qli_lock); if (qlip->qli_item.li_buf) { bp = qlip->qli_item.li_buf; qlip->qli_item.li_buf = NULL; } spin_unlock(&qlip->qli_lock); if (bp) { list_del_init(&qlip->qli_item.li_bio_list); xfs_buf_rele(bp); } } /* * This is called to free all the memory associated with a dquot */ void xfs_qm_dqdestroy( struct xfs_dquot *dqp) { ASSERT(list_empty(&dqp->q_lru)); ASSERT(dqp->q_logitem.qli_item.li_buf == NULL); kvfree(dqp->q_logitem.qli_item.li_lv_shadow); mutex_destroy(&dqp->q_qlock); XFS_STATS_DEC(dqp->q_mount, xs_qm_dquot); kmem_cache_free(xfs_dquot_cache, dqp); } /* * If default limits are in force, push them into the dquot now. * We overwrite the dquot limits only if they are zero and this * is not the root dquot. */ void xfs_qm_adjust_dqlimits( struct xfs_dquot *dq) { struct xfs_mount *mp = dq->q_mount; struct xfs_quotainfo *q = mp->m_quotainfo; struct xfs_def_quota *defq; int prealloc = 0; ASSERT(dq->q_id); defq = xfs_get_defquota(q, xfs_dquot_type(dq)); if (!dq->q_blk.softlimit) { dq->q_blk.softlimit = defq->blk.soft; prealloc = 1; } if (!dq->q_blk.hardlimit) { dq->q_blk.hardlimit = defq->blk.hard; prealloc = 1; } if (!dq->q_ino.softlimit) dq->q_ino.softlimit = defq->ino.soft; if (!dq->q_ino.hardlimit) dq->q_ino.hardlimit = defq->ino.hard; if (!dq->q_rtb.softlimit) dq->q_rtb.softlimit = defq->rtb.soft; if (!dq->q_rtb.hardlimit) dq->q_rtb.hardlimit = defq->rtb.hard; if (prealloc) xfs_dquot_set_prealloc_limits(dq); } /* Set the expiration time of a quota's grace period. */ time64_t xfs_dquot_set_timeout( struct xfs_mount *mp, time64_t timeout) { struct xfs_quotainfo *qi = mp->m_quotainfo; return clamp_t(time64_t, timeout, qi->qi_expiry_min, qi->qi_expiry_max); } /* Set the length of the default grace period. */ time64_t xfs_dquot_set_grace_period( time64_t grace) { return clamp_t(time64_t, grace, XFS_DQ_GRACE_MIN, XFS_DQ_GRACE_MAX); } /* * Determine if this quota counter is over either limit and set the quota * timers as appropriate. */ static inline void xfs_qm_adjust_res_timer( struct xfs_mount *mp, struct xfs_dquot_res *res, struct xfs_quota_limits *qlim) { ASSERT(res->hardlimit == 0 || res->softlimit <= res->hardlimit); if ((res->softlimit && res->count > res->softlimit) || (res->hardlimit && res->count > res->hardlimit)) { if (res->timer == 0) res->timer = xfs_dquot_set_timeout(mp, ktime_get_real_seconds() + qlim->time); } else { res->timer = 0; } } /* * Check the limits and timers of a dquot and start or reset timers * if necessary. * This gets called even when quota enforcement is OFF, which makes our * life a little less complicated. (We just don't reject any quota * reservations in that case, when enforcement is off). * We also return 0 as the values of the timers in Q_GETQUOTA calls, when * enforcement's off. * In contrast, warnings are a little different in that they don't * 'automatically' get started when limits get exceeded. They do * get reset to zero, however, when we find the count to be under * the soft limit (they are only ever set non-zero via userspace). */ void xfs_qm_adjust_dqtimers( struct xfs_dquot *dq) { struct xfs_mount *mp = dq->q_mount; struct xfs_quotainfo *qi = mp->m_quotainfo; struct xfs_def_quota *defq; ASSERT(dq->q_id); defq = xfs_get_defquota(qi, xfs_dquot_type(dq)); xfs_qm_adjust_res_timer(dq->q_mount, &dq->q_blk, &defq->blk); xfs_qm_adjust_res_timer(dq->q_mount, &dq->q_ino, &defq->ino); xfs_qm_adjust_res_timer(dq->q_mount, &dq->q_rtb, &defq->rtb); } /* * initialize a buffer full of dquots and log the whole thing */ void xfs_qm_init_dquot_blk( struct xfs_trans *tp, xfs_dqid_t id, xfs_dqtype_t type, struct xfs_buf *bp) { struct xfs_mount *mp = tp->t_mountp; struct xfs_quotainfo *q = mp->m_quotainfo; struct xfs_dqblk *d; xfs_dqid_t curid; unsigned int qflag; unsigned int blftype; int i; ASSERT(tp); ASSERT(xfs_buf_islocked(bp)); switch (type) { case XFS_DQTYPE_USER: qflag = XFS_UQUOTA_CHKD; blftype = XFS_BLF_UDQUOT_BUF; break; case XFS_DQTYPE_PROJ: qflag = XFS_PQUOTA_CHKD; blftype = XFS_BLF_PDQUOT_BUF; break; case XFS_DQTYPE_GROUP: qflag = XFS_GQUOTA_CHKD; blftype = XFS_BLF_GDQUOT_BUF; break; default: ASSERT(0); return; } d = bp->b_addr; /* * ID of the first dquot in the block - id's are zero based. */ curid = id - (id % q->qi_dqperchunk); memset(d, 0, BBTOB(q->qi_dqchunklen)); for (i = 0; i < q->qi_dqperchunk; i++, d++, curid++) { d->dd_diskdq.d_magic = cpu_to_be16(XFS_DQUOT_MAGIC); d->dd_diskdq.d_version = XFS_DQUOT_VERSION; d->dd_diskdq.d_id = cpu_to_be32(curid); d->dd_diskdq.d_type = type; if (curid > 0 && xfs_has_bigtime(mp)) d->dd_diskdq.d_type |= XFS_DQTYPE_BIGTIME; if (xfs_has_crc(mp)) { uuid_copy(&d->dd_uuid, &mp->m_sb.sb_meta_uuid); xfs_update_cksum((char *)d, sizeof(struct xfs_dqblk), XFS_DQUOT_CRC_OFF); } } xfs_trans_dquot_buf(tp, bp, blftype); /* * quotacheck uses delayed writes to update all the dquots on disk in an * efficient manner instead of logging the individual dquot changes as * they are made. However if we log the buffer allocated here and crash * after quotacheck while the logged initialisation is still in the * active region of the log, log recovery can replay the dquot buffer * initialisation over the top of the checked dquots and corrupt quota * accounting. * * To avoid this problem, quotacheck cannot log the initialised buffer. * We must still dirty the buffer and write it back before the * allocation transaction clears the log. Therefore, mark the buffer as * ordered instead of logging it directly. This is safe for quotacheck * because it detects and repairs allocated but initialized dquot blocks * in the quota inodes. */ if (!(mp->m_qflags & qflag)) xfs_trans_ordered_buf(tp, bp); else xfs_trans_log_buf(tp, bp, 0, BBTOB(q->qi_dqchunklen) - 1); } static void xfs_dquot_set_prealloc( struct xfs_dquot_pre *pre, const struct xfs_dquot_res *res) { xfs_qcnt_t space; pre->q_prealloc_hi_wmark = res->hardlimit; pre->q_prealloc_lo_wmark = res->softlimit; space = div_u64(pre->q_prealloc_hi_wmark, 100); if (!pre->q_prealloc_lo_wmark) pre->q_prealloc_lo_wmark = space * 95; pre->q_low_space[XFS_QLOWSP_1_PCNT] = space; pre->q_low_space[XFS_QLOWSP_3_PCNT] = space * 3; pre->q_low_space[XFS_QLOWSP_5_PCNT] = space * 5; } /* * Initialize the dynamic speculative preallocation thresholds. The lo/hi * watermarks correspond to the soft and hard limits by default. If a soft limit * is not specified, we use 95% of the hard limit. */ void xfs_dquot_set_prealloc_limits(struct xfs_dquot *dqp) { xfs_dquot_set_prealloc(&dqp->q_blk_prealloc, &dqp->q_blk); xfs_dquot_set_prealloc(&dqp->q_rtb_prealloc, &dqp->q_rtb); } /* * Ensure that the given in-core dquot has a buffer on disk backing it, and * return the buffer locked and held. This is called when the bmapi finds a * hole. */ STATIC int xfs_dquot_disk_alloc( struct xfs_dquot *dqp, struct xfs_buf **bpp) { struct xfs_bmbt_irec map; struct xfs_trans *tp; struct xfs_mount *mp = dqp->q_mount; struct xfs_buf *bp; xfs_dqtype_t qtype = xfs_dquot_type(dqp); struct xfs_inode *quotip = xfs_quota_inode(mp, qtype); int nmaps = 1; int error; trace_xfs_dqalloc(dqp); error = xfs_trans_alloc(mp, &M_RES(mp)->tr_qm_dqalloc, XFS_QM_DQALLOC_SPACE_RES(mp), 0, 0, &tp); if (error) return error; xfs_ilock(quotip, XFS_ILOCK_EXCL); xfs_trans_ijoin(tp, quotip, 0); if (!xfs_this_quota_on(dqp->q_mount, qtype)) { /* * Return if this type of quotas is turned off while we didn't * have an inode lock */ error = -ESRCH; goto err_cancel; } error = xfs_iext_count_extend(tp, quotip, XFS_DATA_FORK, XFS_IEXT_ADD_NOSPLIT_CNT); if (error) goto err_cancel; /* Create the block mapping. */ error = xfs_bmapi_write(tp, quotip, dqp->q_fileoffset, XFS_DQUOT_CLUSTER_SIZE_FSB, XFS_BMAPI_METADATA, 0, &map, &nmaps); if (error) goto err_cancel; ASSERT(map.br_blockcount == XFS_DQUOT_CLUSTER_SIZE_FSB); ASSERT((map.br_startblock != DELAYSTARTBLOCK) && (map.br_startblock != HOLESTARTBLOCK)); /* * Keep track of the blkno to save a lookup later */ dqp->q_blkno = XFS_FSB_TO_DADDR(mp, map.br_startblock); /* now we can just get the buffer (there's nothing to read yet) */ error = xfs_trans_get_buf(tp, mp->m_ddev_targp, dqp->q_blkno, mp->m_quotainfo->qi_dqchunklen, 0, &bp); if (error) goto err_cancel; bp->b_ops = &xfs_dquot_buf_ops; /* * Make a chunk of dquots out of this buffer and log * the entire thing. */ xfs_qm_init_dquot_blk(tp, dqp->q_id, qtype, bp); xfs_buf_set_ref(bp, XFS_DQUOT_REF); /* * Hold the buffer and join it to the dfops so that we'll still own * the buffer when we return to the caller. The buffer disposal on * error must be paid attention to very carefully, as it has been * broken since commit efa092f3d4c6 "[XFS] Fixes a bug in the quota * code when allocating a new dquot record" in 2005, and the later * conversion to xfs_defer_ops in commit 310a75a3c6c747 failed to keep * the buffer locked across the _defer_finish call. We can now do * this correctly with xfs_defer_bjoin. * * Above, we allocated a disk block for the dquot information and used * get_buf to initialize the dquot. If the _defer_finish fails, the old * transaction is gone but the new buffer is not joined or held to any * transaction, so we must _buf_relse it. * * If everything succeeds, the caller of this function is returned a * buffer that is locked and held to the transaction. The caller * is responsible for unlocking any buffer passed back, either * manually or by committing the transaction. On error, the buffer is * released and not passed back. * * Keep the quota inode ILOCKed until after the transaction commit to * maintain the atomicity of bmap/rmap updates. */ xfs_trans_bhold(tp, bp); error = xfs_trans_commit(tp); xfs_iunlock(quotip, XFS_ILOCK_EXCL); if (error) { xfs_buf_relse(bp); return error; } *bpp = bp; return 0; err_cancel: xfs_trans_cancel(tp); xfs_iunlock(quotip, XFS_ILOCK_EXCL); return error; } /* * Read in the in-core dquot's on-disk metadata and return the buffer. * Returns ENOENT to signal a hole. */ STATIC int xfs_dquot_disk_read( struct xfs_mount *mp, struct xfs_dquot *dqp, struct xfs_buf **bpp) { struct xfs_bmbt_irec map; struct xfs_buf *bp; xfs_dqtype_t qtype = xfs_dquot_type(dqp); struct xfs_inode *quotip = xfs_quota_inode(mp, qtype); uint lock_mode; int nmaps = 1; int error; lock_mode = xfs_ilock_data_map_shared(quotip); if (!xfs_this_quota_on(mp, qtype)) { /* * Return if this type of quotas is turned off while we * didn't have the quota inode lock. */ xfs_iunlock(quotip, lock_mode); return -ESRCH; } /* * Find the block map; no allocations yet */ error = xfs_bmapi_read(quotip, dqp->q_fileoffset, XFS_DQUOT_CLUSTER_SIZE_FSB, &map, &nmaps, 0); xfs_iunlock(quotip, lock_mode); if (error) return error; ASSERT(nmaps == 1); ASSERT(map.br_blockcount >= 1); ASSERT(map.br_startblock != DELAYSTARTBLOCK); if (map.br_startblock == HOLESTARTBLOCK) return -ENOENT; trace_xfs_dqtobp_read(dqp); /* * store the blkno etc so that we don't have to do the * mapping all the time */ dqp->q_blkno = XFS_FSB_TO_DADDR(mp, map.br_startblock); error = xfs_trans_read_buf(mp, NULL, mp->m_ddev_targp, dqp->q_blkno, mp->m_quotainfo->qi_dqchunklen, 0, &bp, &xfs_dquot_buf_ops); if (xfs_metadata_is_sick(error)) xfs_dquot_mark_sick(dqp); if (error) { ASSERT(bp == NULL); return error; } ASSERT(xfs_buf_islocked(bp)); xfs_buf_set_ref(bp, XFS_DQUOT_REF); *bpp = bp; return 0; } /* Allocate and initialize everything we need for an incore dquot. */ STATIC struct xfs_dquot * xfs_dquot_alloc( struct xfs_mount *mp, xfs_dqid_t id, xfs_dqtype_t type) { struct xfs_dquot *dqp; dqp = kmem_cache_zalloc(xfs_dquot_cache, GFP_KERNEL | __GFP_NOFAIL); dqp->q_type = type; dqp->q_id = id; dqp->q_mount = mp; INIT_LIST_HEAD(&dqp->q_lru); mutex_init(&dqp->q_qlock); init_waitqueue_head(&dqp->q_pinwait); dqp->q_fileoffset = (xfs_fileoff_t)id / mp->m_quotainfo->qi_dqperchunk; /* * Offset of dquot in the (fixed sized) dquot chunk. */ dqp->q_bufoffset = (id % mp->m_quotainfo->qi_dqperchunk) * sizeof(struct xfs_dqblk); /* * Because we want to use a counting completion, complete * the flush completion once to allow a single access to * the flush completion without blocking. */ init_completion(&dqp->q_flush); complete(&dqp->q_flush); /* * Make sure group quotas have a different lock class than user * quotas. */ switch (type) { case XFS_DQTYPE_USER: /* uses the default lock class */ break; case XFS_DQTYPE_GROUP: lockdep_set_class(&dqp->q_qlock, &xfs_dquot_group_class); break; case XFS_DQTYPE_PROJ: lockdep_set_class(&dqp->q_qlock, &xfs_dquot_project_class); break; default: ASSERT(0); break; } xfs_qm_dquot_logitem_init(dqp); XFS_STATS_INC(mp, xs_qm_dquot); return dqp; } /* Check the ondisk dquot's id and type match what the incore dquot expects. */ static bool xfs_dquot_check_type( struct xfs_dquot *dqp, struct xfs_disk_dquot *ddqp) { uint8_t ddqp_type; uint8_t dqp_type; ddqp_type = ddqp->d_type & XFS_DQTYPE_REC_MASK; dqp_type = xfs_dquot_type(dqp); if (be32_to_cpu(ddqp->d_id) != dqp->q_id) return false; /* * V5 filesystems always expect an exact type match. V4 filesystems * expect an exact match for user dquots and for non-root group and * project dquots. */ if (xfs_has_crc(dqp->q_mount) || dqp_type == XFS_DQTYPE_USER || dqp->q_id != 0) return ddqp_type == dqp_type; /* * V4 filesystems support either group or project quotas, but not both * at the same time. The non-user quota file can be switched between * group and project quota uses depending on the mount options, which * means that we can encounter the other type when we try to load quota * defaults. Quotacheck will soon reset the entire quota file * (including the root dquot) anyway, but don't log scary corruption * reports to dmesg. */ return ddqp_type == XFS_DQTYPE_GROUP || ddqp_type == XFS_DQTYPE_PROJ; } /* Copy the in-core quota fields in from the on-disk buffer. */ STATIC int xfs_dquot_from_disk( struct xfs_dquot *dqp, struct xfs_buf *bp) { struct xfs_dqblk *dqb = xfs_buf_offset(bp, dqp->q_bufoffset); struct xfs_disk_dquot *ddqp = &dqb->dd_diskdq; /* * Ensure that we got the type and ID we were looking for. * Everything else was checked by the dquot buffer verifier. */ if (!xfs_dquot_check_type(dqp, ddqp)) { xfs_alert_tag(bp->b_mount, XFS_PTAG_VERIFIER_ERROR, "Metadata corruption detected at %pS, quota %u", __this_address, dqp->q_id); xfs_alert(bp->b_mount, "Unmount and run xfs_repair"); xfs_dquot_mark_sick(dqp); return -EFSCORRUPTED; } /* copy everything from disk dquot to the incore dquot */ dqp->q_type = ddqp->d_type; dqp->q_blk.hardlimit = be64_to_cpu(ddqp->d_blk_hardlimit); dqp->q_blk.softlimit = be64_to_cpu(ddqp->d_blk_softlimit); dqp->q_ino.hardlimit = be64_to_cpu(ddqp->d_ino_hardlimit); dqp->q_ino.softlimit = be64_to_cpu(ddqp->d_ino_softlimit); dqp->q_rtb.hardlimit = be64_to_cpu(ddqp->d_rtb_hardlimit); dqp->q_rtb.softlimit = be64_to_cpu(ddqp->d_rtb_softlimit); dqp->q_blk.count = be64_to_cpu(ddqp->d_bcount); dqp->q_ino.count = be64_to_cpu(ddqp->d_icount); dqp->q_rtb.count = be64_to_cpu(ddqp->d_rtbcount); dqp->q_blk.timer = xfs_dquot_from_disk_ts(ddqp, ddqp->d_btimer); dqp->q_ino.timer = xfs_dquot_from_disk_ts(ddqp, ddqp->d_itimer); dqp->q_rtb.timer = xfs_dquot_from_disk_ts(ddqp, ddqp->d_rtbtimer); /* * Reservation counters are defined as reservation plus current usage * to avoid having to add every time. */ dqp->q_blk.reserved = dqp->q_blk.count; dqp->q_ino.reserved = dqp->q_ino.count; dqp->q_rtb.reserved = dqp->q_rtb.count; /* initialize the dquot speculative prealloc thresholds */ xfs_dquot_set_prealloc_limits(dqp); return 0; } /* Copy the in-core quota fields into the on-disk buffer. */ void xfs_dquot_to_disk( struct xfs_disk_dquot *ddqp, struct xfs_dquot *dqp) { ddqp->d_magic = cpu_to_be16(XFS_DQUOT_MAGIC); ddqp->d_version = XFS_DQUOT_VERSION; ddqp->d_type = dqp->q_type; ddqp->d_id = cpu_to_be32(dqp->q_id); ddqp->d_pad0 = 0; ddqp->d_pad = 0; ddqp->d_blk_hardlimit = cpu_to_be64(dqp->q_blk.hardlimit); ddqp->d_blk_softlimit = cpu_to_be64(dqp->q_blk.softlimit); ddqp->d_ino_hardlimit = cpu_to_be64(dqp->q_ino.hardlimit); ddqp->d_ino_softlimit = cpu_to_be64(dqp->q_ino.softlimit); ddqp->d_rtb_hardlimit = cpu_to_be64(dqp->q_rtb.hardlimit); ddqp->d_rtb_softlimit = cpu_to_be64(dqp->q_rtb.softlimit); ddqp->d_bcount = cpu_to_be64(dqp->q_blk.count); ddqp->d_icount = cpu_to_be64(dqp->q_ino.count); ddqp->d_rtbcount = cpu_to_be64(dqp->q_rtb.count); ddqp->d_bwarns = 0; ddqp->d_iwarns = 0; ddqp->d_rtbwarns = 0; ddqp->d_btimer = xfs_dquot_to_disk_ts(dqp, dqp->q_blk.timer); ddqp->d_itimer = xfs_dquot_to_disk_ts(dqp, dqp->q_ino.timer); ddqp->d_rtbtimer = xfs_dquot_to_disk_ts(dqp, dqp->q_rtb.timer); } /* * Read in the ondisk dquot using dqtobp() then copy it to an incore version, * and release the buffer immediately. If @can_alloc is true, fill any * holes in the on-disk metadata. */ static int xfs_qm_dqread( struct xfs_mount *mp, xfs_dqid_t id, xfs_dqtype_t type, bool can_alloc, struct xfs_dquot **dqpp) { struct xfs_dquot *dqp; struct xfs_buf *bp; int error; dqp = xfs_dquot_alloc(mp, id, type); trace_xfs_dqread(dqp); /* Try to read the buffer, allocating if necessary. */ error = xfs_dquot_disk_read(mp, dqp, &bp); if (error == -ENOENT && can_alloc) error = xfs_dquot_disk_alloc(dqp, &bp); if (error) goto err; /* * At this point we should have a clean locked buffer. Copy the data * to the incore dquot and release the buffer since the incore dquot * has its own locking protocol so we needn't tie up the buffer any * further. */ ASSERT(xfs_buf_islocked(bp)); error = xfs_dquot_from_disk(dqp, bp); xfs_buf_relse(bp); if (error) goto err; *dqpp = dqp; return error; err: trace_xfs_dqread_fail(dqp); xfs_qm_dqdestroy(dqp); *dqpp = NULL; return error; } /* * Advance to the next id in the current chunk, or if at the * end of the chunk, skip ahead to first id in next allocated chunk * using the SEEK_DATA interface. */ static int xfs_dq_get_next_id( struct xfs_mount *mp, xfs_dqtype_t type, xfs_dqid_t *id) { struct xfs_inode *quotip = xfs_quota_inode(mp, type); xfs_dqid_t next_id = *id + 1; /* simple advance */ uint lock_flags; struct xfs_bmbt_irec got; struct xfs_iext_cursor cur; xfs_fsblock_t start; int error = 0; /* If we'd wrap past the max ID, stop */ if (next_id < *id) return -ENOENT; /* If new ID is within the current chunk, advancing it sufficed */ if (next_id % mp->m_quotainfo->qi_dqperchunk) { *id = next_id; return 0; } /* Nope, next_id is now past the current chunk, so find the next one */ start = (xfs_fsblock_t)next_id / mp->m_quotainfo->qi_dqperchunk; lock_flags = xfs_ilock_data_map_shared(quotip); error = xfs_iread_extents(NULL, quotip, XFS_DATA_FORK); if (error) return error; if (xfs_iext_lookup_extent(quotip, "ip->i_df, start, &cur, &got)) { /* contiguous chunk, bump startoff for the id calculation */ if (got.br_startoff < start) got.br_startoff = start; *id = got.br_startoff * mp->m_quotainfo->qi_dqperchunk; } else { error = -ENOENT; } xfs_iunlock(quotip, lock_flags); return error; } /* * Look up the dquot in the in-core cache. If found, the dquot is returned * locked and ready to go. */ static struct xfs_dquot * xfs_qm_dqget_cache_lookup( struct xfs_mount *mp, struct xfs_quotainfo *qi, struct radix_tree_root *tree, xfs_dqid_t id) { struct xfs_dquot *dqp; restart: mutex_lock(&qi->qi_tree_lock); dqp = radix_tree_lookup(tree, id); if (!dqp) { mutex_unlock(&qi->qi_tree_lock); XFS_STATS_INC(mp, xs_qm_dqcachemisses); return NULL; } xfs_dqlock(dqp); if (dqp->q_flags & XFS_DQFLAG_FREEING) { xfs_dqunlock(dqp); mutex_unlock(&qi->qi_tree_lock); trace_xfs_dqget_freeing(dqp); delay(1); goto restart; } dqp->q_nrefs++; mutex_unlock(&qi->qi_tree_lock); trace_xfs_dqget_hit(dqp); XFS_STATS_INC(mp, xs_qm_dqcachehits); return dqp; } /* * Try to insert a new dquot into the in-core cache. If an error occurs the * caller should throw away the dquot and start over. Otherwise, the dquot * is returned locked (and held by the cache) as if there had been a cache * hit. * * The insert needs to be done under memalloc_nofs context because the radix * tree can do memory allocation during insert. The qi->qi_tree_lock is taken in * memory reclaim when freeing unused dquots, so we cannot have the radix tree * node allocation recursing into filesystem reclaim whilst we hold the * qi_tree_lock. */ static int xfs_qm_dqget_cache_insert( struct xfs_mount *mp, struct xfs_quotainfo *qi, struct radix_tree_root *tree, xfs_dqid_t id, struct xfs_dquot *dqp) { unsigned int nofs_flags; int error; nofs_flags = memalloc_nofs_save(); mutex_lock(&qi->qi_tree_lock); error = radix_tree_insert(tree, id, dqp); if (unlikely(error)) { /* Duplicate found! Caller must try again. */ trace_xfs_dqget_dup(dqp); goto out_unlock; } /* Return a locked dquot to the caller, with a reference taken. */ xfs_dqlock(dqp); dqp->q_nrefs = 1; qi->qi_dquots++; out_unlock: mutex_unlock(&qi->qi_tree_lock); memalloc_nofs_restore(nofs_flags); return error; } /* Check our input parameters. */ static int xfs_qm_dqget_checks( struct xfs_mount *mp, xfs_dqtype_t type) { switch (type) { case XFS_DQTYPE_USER: if (!XFS_IS_UQUOTA_ON(mp)) return -ESRCH; return 0; case XFS_DQTYPE_GROUP: if (!XFS_IS_GQUOTA_ON(mp)) return -ESRCH; return 0; case XFS_DQTYPE_PROJ: if (!XFS_IS_PQUOTA_ON(mp)) return -ESRCH; return 0; default: WARN_ON_ONCE(0); return -EINVAL; } } /* * Given the file system, id, and type (UDQUOT/GDQUOT/PDQUOT), return a * locked dquot, doing an allocation (if requested) as needed. */ int xfs_qm_dqget( struct xfs_mount *mp, xfs_dqid_t id, xfs_dqtype_t type, bool can_alloc, struct xfs_dquot **O_dqpp) { struct xfs_quotainfo *qi = mp->m_quotainfo; struct radix_tree_root *tree = xfs_dquot_tree(qi, type); struct xfs_dquot *dqp; int error; error = xfs_qm_dqget_checks(mp, type); if (error) return error; restart: dqp = xfs_qm_dqget_cache_lookup(mp, qi, tree, id); if (dqp) { *O_dqpp = dqp; return 0; } error = xfs_qm_dqread(mp, id, type, can_alloc, &dqp); if (error) return error; error = xfs_qm_dqget_cache_insert(mp, qi, tree, id, dqp); if (error) { /* * Duplicate found. Just throw away the new dquot and start * over. */ xfs_qm_dqdestroy(dqp); XFS_STATS_INC(mp, xs_qm_dquot_dups); goto restart; } trace_xfs_dqget_miss(dqp); *O_dqpp = dqp; return 0; } /* * Given a dquot id and type, read and initialize a dquot from the on-disk * metadata. This function is only for use during quota initialization so * it ignores the dquot cache assuming that the dquot shrinker isn't set up. * The caller is responsible for _qm_dqdestroy'ing the returned dquot. */ int xfs_qm_dqget_uncached( struct xfs_mount *mp, xfs_dqid_t id, xfs_dqtype_t type, struct xfs_dquot **dqpp) { int error; error = xfs_qm_dqget_checks(mp, type); if (error) return error; return xfs_qm_dqread(mp, id, type, 0, dqpp); } /* Return the quota id for a given inode and type. */ xfs_dqid_t xfs_qm_id_for_quotatype( struct xfs_inode *ip, xfs_dqtype_t type) { switch (type) { case XFS_DQTYPE_USER: return i_uid_read(VFS_I(ip)); case XFS_DQTYPE_GROUP: return i_gid_read(VFS_I(ip)); case XFS_DQTYPE_PROJ: return ip->i_projid; } ASSERT(0); return 0; } /* * Return the dquot for a given inode and type. If @can_alloc is true, then * allocate blocks if needed. The inode's ILOCK must be held and it must not * have already had an inode attached. */ int xfs_qm_dqget_inode( struct xfs_inode *ip, xfs_dqtype_t type, bool can_alloc, struct xfs_dquot **O_dqpp) { struct xfs_mount *mp = ip->i_mount; struct xfs_quotainfo *qi = mp->m_quotainfo; struct radix_tree_root *tree = xfs_dquot_tree(qi, type); struct xfs_dquot *dqp; xfs_dqid_t id; int error; error = xfs_qm_dqget_checks(mp, type); if (error) return error; xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); ASSERT(xfs_inode_dquot(ip, type) == NULL); ASSERT(!xfs_is_metadir_inode(ip)); id = xfs_qm_id_for_quotatype(ip, type); restart: dqp = xfs_qm_dqget_cache_lookup(mp, qi, tree, id); if (dqp) { *O_dqpp = dqp; return 0; } /* * Dquot cache miss. We don't want to keep the inode lock across * a (potential) disk read. Also we don't want to deal with the lock * ordering between quotainode and this inode. OTOH, dropping the inode * lock here means dealing with a chown that can happen before * we re-acquire the lock. */ xfs_iunlock(ip, XFS_ILOCK_EXCL); error = xfs_qm_dqread(mp, id, type, can_alloc, &dqp); xfs_ilock(ip, XFS_ILOCK_EXCL); if (error) return error; /* * A dquot could be attached to this inode by now, since we had * dropped the ilock. */ if (xfs_this_quota_on(mp, type)) { struct xfs_dquot *dqp1; dqp1 = xfs_inode_dquot(ip, type); if (dqp1) { xfs_qm_dqdestroy(dqp); dqp = dqp1; xfs_dqlock(dqp); goto dqret; } } else { /* inode stays locked on return */ xfs_qm_dqdestroy(dqp); return -ESRCH; } error = xfs_qm_dqget_cache_insert(mp, qi, tree, id, dqp); if (error) { /* * Duplicate found. Just throw away the new dquot and start * over. */ xfs_qm_dqdestroy(dqp); XFS_STATS_INC(mp, xs_qm_dquot_dups); goto restart; } dqret: xfs_assert_ilocked(ip, XFS_ILOCK_EXCL); trace_xfs_dqget_miss(dqp); *O_dqpp = dqp; return 0; } /* * Starting at @id and progressing upwards, look for an initialized incore * dquot, lock it, and return it. */ int xfs_qm_dqget_next( struct xfs_mount *mp, xfs_dqid_t id, xfs_dqtype_t type, struct xfs_dquot **dqpp) { struct xfs_dquot *dqp; int error = 0; *dqpp = NULL; for (; !error; error = xfs_dq_get_next_id(mp, type, &id)) { error = xfs_qm_dqget(mp, id, type, false, &dqp); if (error == -ENOENT) continue; else if (error != 0) break; if (!XFS_IS_DQUOT_UNINITIALIZED(dqp)) { *dqpp = dqp; return 0; } xfs_qm_dqput(dqp); } return error; } /* * Release a reference to the dquot (decrement ref-count) and unlock it. * * If there is a group quota attached to this dquot, carefully release that * too without tripping over deadlocks'n'stuff. */ void xfs_qm_dqput( struct xfs_dquot *dqp) { ASSERT(dqp->q_nrefs > 0); ASSERT(XFS_DQ_IS_LOCKED(dqp)); trace_xfs_dqput(dqp); if (--dqp->q_nrefs == 0) { struct xfs_quotainfo *qi = dqp->q_mount->m_quotainfo; trace_xfs_dqput_free(dqp); if (list_lru_add_obj(&qi->qi_lru, &dqp->q_lru)) XFS_STATS_INC(dqp->q_mount, xs_qm_dquot_unused); } xfs_dqunlock(dqp); } /* * Release a dquot. Flush it if dirty, then dqput() it. * dquot must not be locked. */ void xfs_qm_dqrele( struct xfs_dquot *dqp) { if (!dqp) return; trace_xfs_dqrele(dqp); xfs_dqlock(dqp); /* * We don't care to flush it if the dquot is dirty here. * That will create stutters that we want to avoid. * Instead we do a delayed write when we try to reclaim * a dirty dquot. Also xfs_sync will take part of the burden... */ xfs_qm_dqput(dqp); } /* * This is the dquot flushing I/O completion routine. It is called * from interrupt level when the buffer containing the dquot is * flushed to disk. It is responsible for removing the dquot logitem * from the AIL if it has not been re-logged, and unlocking the dquot's * flush lock. This behavior is very similar to that of inodes.. */ static void xfs_qm_dqflush_done( struct xfs_log_item *lip) { struct xfs_dq_logitem *qlip = container_of(lip, struct xfs_dq_logitem, qli_item); struct xfs_dquot *dqp = qlip->qli_dquot; struct xfs_ail *ailp = lip->li_ailp; struct xfs_buf *bp = NULL; xfs_lsn_t tail_lsn; /* * We only want to pull the item from the AIL if its * location in the log has not changed since we started the flush. * Thus, we only bother if the dquot's lsn has * not changed. First we check the lsn outside the lock * since it's cheaper, and then we recheck while * holding the lock before removing the dquot from the AIL. */ if (test_bit(XFS_LI_IN_AIL, &lip->li_flags) && (lip->li_lsn == qlip->qli_flush_lsn || test_bit(XFS_LI_FAILED, &lip->li_flags))) { spin_lock(&ailp->ail_lock); xfs_clear_li_failed(lip); if (lip->li_lsn == qlip->qli_flush_lsn) { /* xfs_ail_update_finish() drops the AIL lock */ tail_lsn = xfs_ail_delete_one(ailp, lip); xfs_ail_update_finish(ailp, tail_lsn); } else { spin_unlock(&ailp->ail_lock); } } /* * If this dquot hasn't been dirtied since initiating the last dqflush, * release the buffer reference. We already unlinked this dquot item * from the buffer. */ spin_lock(&qlip->qli_lock); if (!qlip->qli_dirty) { bp = lip->li_buf; lip->li_buf = NULL; } spin_unlock(&qlip->qli_lock); if (bp) xfs_buf_rele(bp); /* * Release the dq's flush lock since we're done with it. */ xfs_dqfunlock(dqp); } void xfs_buf_dquot_iodone( struct xfs_buf *bp) { struct xfs_log_item *lip, *n; list_for_each_entry_safe(lip, n, &bp->b_li_list, li_bio_list) { list_del_init(&lip->li_bio_list); xfs_qm_dqflush_done(lip); } } void xfs_buf_dquot_io_fail( struct xfs_buf *bp) { struct xfs_log_item *lip; spin_lock(&bp->b_mount->m_ail->ail_lock); list_for_each_entry(lip, &bp->b_li_list, li_bio_list) set_bit(XFS_LI_FAILED, &lip->li_flags); spin_unlock(&bp->b_mount->m_ail->ail_lock); } /* Check incore dquot for errors before we flush. */ static xfs_failaddr_t xfs_qm_dqflush_check( struct xfs_dquot *dqp) { xfs_dqtype_t type = xfs_dquot_type(dqp); if (type != XFS_DQTYPE_USER && type != XFS_DQTYPE_GROUP && type != XFS_DQTYPE_PROJ) return __this_address; if (dqp->q_id == 0) return NULL; if (dqp->q_blk.softlimit && dqp->q_blk.count > dqp->q_blk.softlimit && !dqp->q_blk.timer) return __this_address; if (dqp->q_ino.softlimit && dqp->q_ino.count > dqp->q_ino.softlimit && !dqp->q_ino.timer) return __this_address; if (dqp->q_rtb.softlimit && dqp->q_rtb.count > dqp->q_rtb.softlimit && !dqp->q_rtb.timer) return __this_address; /* bigtime flag should never be set on root dquots */ if (dqp->q_type & XFS_DQTYPE_BIGTIME) { if (!xfs_has_bigtime(dqp->q_mount)) return __this_address; if (dqp->q_id == 0) return __this_address; } return NULL; } /* * Get the buffer containing the on-disk dquot. * * Requires dquot flush lock, will clear the dirty flag, delete the quota log * item from the AIL, and shut down the system if something goes wrong. */ static int xfs_dquot_read_buf( struct xfs_trans *tp, struct xfs_dquot *dqp, struct xfs_buf **bpp) { struct xfs_mount *mp = dqp->q_mount; struct xfs_buf *bp = NULL; int error; error = xfs_trans_read_buf(mp, tp, mp->m_ddev_targp, dqp->q_blkno, mp->m_quotainfo->qi_dqchunklen, 0, &bp, &xfs_dquot_buf_ops); if (xfs_metadata_is_sick(error)) xfs_dquot_mark_sick(dqp); if (error) goto out_abort; *bpp = bp; return 0; out_abort: dqp->q_flags &= ~XFS_DQFLAG_DIRTY; xfs_trans_ail_delete(&dqp->q_logitem.qli_item, 0); xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); return error; } /* * Attach a dquot buffer to this dquot to avoid allocating a buffer during a * dqflush, since dqflush can be called from reclaim context. */ int xfs_dquot_attach_buf( struct xfs_trans *tp, struct xfs_dquot *dqp) { struct xfs_dq_logitem *qlip = &dqp->q_logitem; struct xfs_log_item *lip = &qlip->qli_item; int error; spin_lock(&qlip->qli_lock); if (!lip->li_buf) { struct xfs_buf *bp = NULL; spin_unlock(&qlip->qli_lock); error = xfs_dquot_read_buf(tp, dqp, &bp); if (error) return error; /* * Attach the dquot to the buffer so that the AIL does not have * to read the dquot buffer to push this item. */ xfs_buf_hold(bp); spin_lock(&qlip->qli_lock); lip->li_buf = bp; xfs_trans_brelse(tp, bp); } qlip->qli_dirty = true; spin_unlock(&qlip->qli_lock); return 0; } /* * Get a new reference the dquot buffer attached to this dquot for a dqflush * operation. * * Returns 0 and a NULL bp if none was attached to the dquot; 0 and a locked * bp; or -EAGAIN if the buffer could not be locked. */ int xfs_dquot_use_attached_buf( struct xfs_dquot *dqp, struct xfs_buf **bpp) { struct xfs_buf *bp = dqp->q_logitem.qli_item.li_buf; /* * A NULL buffer can happen if the dquot dirty flag was set but the * filesystem shut down before transaction commit happened. In that * case we're not going to flush anyway. */ if (!bp) { ASSERT(xfs_is_shutdown(dqp->q_mount)); *bpp = NULL; return 0; } if (!xfs_buf_trylock(bp)) return -EAGAIN; xfs_buf_hold(bp); *bpp = bp; return 0; } /* * Write a modified dquot to disk. * The dquot must be locked and the flush lock too taken by caller. * The flush lock will not be unlocked until the dquot reaches the disk, * but the dquot is free to be unlocked and modified by the caller * in the interim. Dquot is still locked on return. This behavior is * identical to that of inodes. */ int xfs_qm_dqflush( struct xfs_dquot *dqp, struct xfs_buf *bp) { struct xfs_mount *mp = dqp->q_mount; struct xfs_dq_logitem *qlip = &dqp->q_logitem; struct xfs_log_item *lip = &qlip->qli_item; struct xfs_dqblk *dqblk; xfs_failaddr_t fa; int error; ASSERT(XFS_DQ_IS_LOCKED(dqp)); ASSERT(!completion_done(&dqp->q_flush)); trace_xfs_dqflush(dqp); xfs_qm_dqunpin_wait(dqp); fa = xfs_qm_dqflush_check(dqp); if (fa) { xfs_alert(mp, "corrupt dquot ID 0x%x in memory at %pS", dqp->q_id, fa); xfs_dquot_mark_sick(dqp); error = -EFSCORRUPTED; goto out_abort; } /* Flush the incore dquot to the ondisk buffer. */ dqblk = xfs_buf_offset(bp, dqp->q_bufoffset); xfs_dquot_to_disk(&dqblk->dd_diskdq, dqp); /* * Clear the dirty field and remember the flush lsn for later use. */ dqp->q_flags &= ~XFS_DQFLAG_DIRTY; /* * We hold the dquot lock, so nobody can dirty it while we're * scheduling the write out. Clear the dirty-since-flush flag. */ spin_lock(&qlip->qli_lock); qlip->qli_dirty = false; spin_unlock(&qlip->qli_lock); xfs_trans_ail_copy_lsn(mp->m_ail, &qlip->qli_flush_lsn, &lip->li_lsn); /* * copy the lsn into the on-disk dquot now while we have the in memory * dquot here. This can't be done later in the write verifier as we * can't get access to the log item at that point in time. * * We also calculate the CRC here so that the on-disk dquot in the * buffer always has a valid CRC. This ensures there is no possibility * of a dquot without an up-to-date CRC getting to disk. */ if (xfs_has_crc(mp)) { dqblk->dd_lsn = cpu_to_be64(lip->li_lsn); xfs_update_cksum((char *)dqblk, sizeof(struct xfs_dqblk), XFS_DQUOT_CRC_OFF); } /* * Attach the dquot to the buffer so that we can remove this dquot from * the AIL and release the flush lock once the dquot is synced to disk. */ bp->b_flags |= _XBF_DQUOTS; list_add_tail(&lip->li_bio_list, &bp->b_li_list); /* * If the buffer is pinned then push on the log so we won't * get stuck waiting in the write for too long. */ if (xfs_buf_ispinned(bp)) { trace_xfs_dqflush_force(dqp); xfs_log_force(mp, 0); } trace_xfs_dqflush_done(dqp); return 0; out_abort: dqp->q_flags &= ~XFS_DQFLAG_DIRTY; xfs_trans_ail_delete(lip, 0); xfs_force_shutdown(mp, SHUTDOWN_CORRUPT_INCORE); xfs_dqfunlock(dqp); return error; } /* * Lock two xfs_dquot structures. * * To avoid deadlocks we always lock the quota structure with * the lowerd id first. */ void xfs_dqlock2( struct xfs_dquot *d1, struct xfs_dquot *d2) { if (d1 && d2) { ASSERT(d1 != d2); if (d1->q_id > d2->q_id) { mutex_lock(&d2->q_qlock); mutex_lock_nested(&d1->q_qlock, XFS_QLOCK_NESTED); } else { mutex_lock(&d1->q_qlock); mutex_lock_nested(&d2->q_qlock, XFS_QLOCK_NESTED); } } else if (d1) { mutex_lock(&d1->q_qlock); } else if (d2) { mutex_lock(&d2->q_qlock); } } static int xfs_dqtrx_cmp( const void *a, const void *b) { const struct xfs_dqtrx *qa = a; const struct xfs_dqtrx *qb = b; if (qa->qt_dquot->q_id > qb->qt_dquot->q_id) return 1; if (qa->qt_dquot->q_id < qb->qt_dquot->q_id) return -1; return 0; } void xfs_dqlockn( struct xfs_dqtrx *q) { unsigned int i; BUILD_BUG_ON(XFS_QM_TRANS_MAXDQS > MAX_LOCKDEP_SUBCLASSES); /* Sort in order of dquot id, do not allow duplicates */ for (i = 0; i < XFS_QM_TRANS_MAXDQS && q[i].qt_dquot != NULL; i++) { unsigned int j; for (j = 0; j < i; j++) ASSERT(q[i].qt_dquot != q[j].qt_dquot); } if (i == 0) return; sort(q, i, sizeof(struct xfs_dqtrx), xfs_dqtrx_cmp, NULL); mutex_lock(&q[0].qt_dquot->q_qlock); for (i = 1; i < XFS_QM_TRANS_MAXDQS && q[i].qt_dquot != NULL; i++) mutex_lock_nested(&q[i].qt_dquot->q_qlock, XFS_QLOCK_NESTED + i - 1); } int __init xfs_qm_init(void) { xfs_dquot_cache = kmem_cache_create("xfs_dquot", sizeof(struct xfs_dquot), 0, 0, NULL); if (!xfs_dquot_cache) goto out; xfs_dqtrx_cache = kmem_cache_create("xfs_dqtrx", sizeof(struct xfs_dquot_acct), 0, 0, NULL); if (!xfs_dqtrx_cache) goto out_free_dquot_cache; return 0; out_free_dquot_cache: kmem_cache_destroy(xfs_dquot_cache); out: return -ENOMEM; } void xfs_qm_exit(void) { kmem_cache_destroy(xfs_dqtrx_cache); kmem_cache_destroy(xfs_dquot_cache); } |
| 2 2 2 7 11 11 5 17 2 20 2 1 1 26 11 15 2 24 24 34 4 1 26 1 1 1 14 1 14 4 4 2 2 2 4 4 4 6 2 4 4 4 2 2 2 13 13 10 10 2 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 | // SPDX-License-Identifier: GPL-2.0-only #include <linux/ceph/ceph_debug.h> #include <linux/backing-dev.h> #include <linux/ctype.h> #include <linux/fs.h> #include <linux/inet.h> #include <linux/in6.h> #include <linux/module.h> #include <linux/mount.h> #include <linux/fs_context.h> #include <linux/fs_parser.h> #include <linux/sched.h> #include <linux/seq_file.h> #include <linux/slab.h> #include <linux/statfs.h> #include <linux/string.h> #include "super.h" #include "mds_client.h" #include "cache.h" #include "crypto.h" #include <linux/ceph/ceph_features.h> #include <linux/ceph/decode.h> #include <linux/ceph/mon_client.h> #include <linux/ceph/auth.h> #include <linux/ceph/debugfs.h> #include <uapi/linux/magic.h> static DEFINE_SPINLOCK(ceph_fsc_lock); static LIST_HEAD(ceph_fsc_list); /* * Ceph superblock operations * * Handle the basics of mounting, unmounting. */ /* * super ops */ static void ceph_put_super(struct super_block *s) { struct ceph_fs_client *fsc = ceph_sb_to_fs_client(s); doutc(fsc->client, "begin\n"); ceph_fscrypt_free_dummy_policy(fsc); ceph_mdsc_close_sessions(fsc->mdsc); doutc(fsc->client, "done\n"); } static int ceph_statfs(struct dentry *dentry, struct kstatfs *buf) { struct ceph_fs_client *fsc = ceph_inode_to_fs_client(d_inode(dentry)); struct ceph_mon_client *monc = &fsc->client->monc; struct ceph_statfs st; int i, err; u64 data_pool; doutc(fsc->client, "begin\n"); if (fsc->mdsc->mdsmap->m_num_data_pg_pools == 1) { data_pool = fsc->mdsc->mdsmap->m_data_pg_pools[0]; } else { data_pool = CEPH_NOPOOL; } err = ceph_monc_do_statfs(monc, data_pool, &st); if (err < 0) return err; /* fill in kstatfs */ buf->f_type = CEPH_SUPER_MAGIC; /* ?? */ /* * Express utilization in terms of large blocks to avoid * overflow on 32-bit machines. */ buf->f_frsize = 1 << CEPH_BLOCK_SHIFT; /* * By default use root quota for stats; fallback to overall filesystem * usage if using 'noquotadf' mount option or if the root dir doesn't * have max_bytes quota set. */ if (ceph_test_mount_opt(fsc, NOQUOTADF) || !ceph_quota_update_statfs(fsc, buf)) { buf->f_blocks = le64_to_cpu(st.kb) >> (CEPH_BLOCK_SHIFT-10); buf->f_bfree = le64_to_cpu(st.kb_avail) >> (CEPH_BLOCK_SHIFT-10); buf->f_bavail = le64_to_cpu(st.kb_avail) >> (CEPH_BLOCK_SHIFT-10); } /* * NOTE: for the time being, we make bsize == frsize to humor * not-yet-ancient versions of glibc that are broken. * Someday, we will probably want to report a real block * size... whatever that may mean for a network file system! */ buf->f_bsize = buf->f_frsize; buf->f_files = le64_to_cpu(st.num_objects); buf->f_ffree = -1; buf->f_namelen = NAME_MAX; /* Must convert the fsid, for consistent values across arches */ buf->f_fsid.val[0] = 0; mutex_lock(&monc->mutex); for (i = 0 ; i < sizeof(monc->monmap->fsid) / sizeof(__le32) ; ++i) buf->f_fsid.val[0] ^= le32_to_cpu(((__le32 *)&monc->monmap->fsid)[i]); mutex_unlock(&monc->mutex); /* fold the fs_cluster_id into the upper bits */ buf->f_fsid.val[1] = monc->fs_cluster_id; doutc(fsc->client, "done\n"); return 0; } static int ceph_sync_fs(struct super_block *sb, int wait) { struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); struct ceph_client *cl = fsc->client; if (!wait) { doutc(cl, "(non-blocking)\n"); ceph_flush_dirty_caps(fsc->mdsc); ceph_flush_cap_releases(fsc->mdsc); doutc(cl, "(non-blocking) done\n"); return 0; } doutc(cl, "(blocking)\n"); ceph_osdc_sync(&fsc->client->osdc); ceph_mdsc_sync(fsc->mdsc); doutc(cl, "(blocking) done\n"); return 0; } /* * mount options */ enum { Opt_wsize, Opt_rsize, Opt_rasize, Opt_caps_wanted_delay_min, Opt_caps_wanted_delay_max, Opt_caps_max, Opt_readdir_max_entries, Opt_readdir_max_bytes, Opt_congestion_kb, /* int args above */ Opt_snapdirname, Opt_mds_namespace, Opt_recover_session, Opt_source, Opt_mon_addr, Opt_test_dummy_encryption, /* string args above */ Opt_dirstat, Opt_rbytes, Opt_asyncreaddir, Opt_dcache, Opt_ino32, Opt_fscache, Opt_poolperm, Opt_require_active_mds, Opt_acl, Opt_quotadf, Opt_copyfrom, Opt_wsync, Opt_pagecache, Opt_sparseread, }; enum ceph_recover_session_mode { ceph_recover_session_no, ceph_recover_session_clean }; static const struct constant_table ceph_param_recover[] = { { "no", ceph_recover_session_no }, { "clean", ceph_recover_session_clean }, {} }; static const struct fs_parameter_spec ceph_mount_parameters[] = { fsparam_flag_no ("acl", Opt_acl), fsparam_flag_no ("asyncreaddir", Opt_asyncreaddir), fsparam_s32 ("caps_max", Opt_caps_max), fsparam_u32 ("caps_wanted_delay_max", Opt_caps_wanted_delay_max), fsparam_u32 ("caps_wanted_delay_min", Opt_caps_wanted_delay_min), fsparam_u32 ("write_congestion_kb", Opt_congestion_kb), fsparam_flag_no ("copyfrom", Opt_copyfrom), fsparam_flag_no ("dcache", Opt_dcache), fsparam_flag_no ("dirstat", Opt_dirstat), fsparam_flag_no ("fsc", Opt_fscache), // fsc|nofsc fsparam_string ("fsc", Opt_fscache), // fsc=... fsparam_flag_no ("ino32", Opt_ino32), fsparam_string ("mds_namespace", Opt_mds_namespace), fsparam_string ("mon_addr", Opt_mon_addr), fsparam_flag_no ("poolperm", Opt_poolperm), fsparam_flag_no ("quotadf", Opt_quotadf), fsparam_u32 ("rasize", Opt_rasize), fsparam_flag_no ("rbytes", Opt_rbytes), fsparam_u32 ("readdir_max_bytes", Opt_readdir_max_bytes), fsparam_u32 ("readdir_max_entries", Opt_readdir_max_entries), fsparam_enum ("recover_session", Opt_recover_session, ceph_param_recover), fsparam_flag_no ("require_active_mds", Opt_require_active_mds), fsparam_u32 ("rsize", Opt_rsize), fsparam_string ("snapdirname", Opt_snapdirname), fsparam_string ("source", Opt_source), fsparam_flag ("test_dummy_encryption", Opt_test_dummy_encryption), fsparam_string ("test_dummy_encryption", Opt_test_dummy_encryption), fsparam_u32 ("wsize", Opt_wsize), fsparam_flag_no ("wsync", Opt_wsync), fsparam_flag_no ("pagecache", Opt_pagecache), fsparam_flag_no ("sparseread", Opt_sparseread), {} }; struct ceph_parse_opts_ctx { struct ceph_options *copts; struct ceph_mount_options *opts; }; /* * Remove adjacent slashes and then the trailing slash, unless it is * the only remaining character. * * E.g. "//dir1////dir2///" --> "/dir1/dir2", "///" --> "/". */ static void canonicalize_path(char *path) { int i, j = 0; for (i = 0; path[i] != '\0'; i++) { if (path[i] != '/' || j < 1 || path[j - 1] != '/') path[j++] = path[i]; } if (j > 1 && path[j - 1] == '/') j--; path[j] = '\0'; } /* * Check if the mds namespace in ceph_mount_options matches * the passed in namespace string. First time match (when * ->mds_namespace is NULL) is treated specially, since * ->mds_namespace needs to be initialized by the caller. */ static int namespace_equals(struct ceph_mount_options *fsopt, const char *namespace, size_t len) { return !(fsopt->mds_namespace && (strlen(fsopt->mds_namespace) != len || strncmp(fsopt->mds_namespace, namespace, len))); } static int ceph_parse_old_source(const char *dev_name, const char *dev_name_end, struct fs_context *fc) { int r; struct ceph_parse_opts_ctx *pctx = fc->fs_private; struct ceph_mount_options *fsopt = pctx->opts; if (*dev_name_end != ':') return invalfc(fc, "separator ':' missing in source"); r = ceph_parse_mon_ips(dev_name, dev_name_end - dev_name, pctx->copts, fc->log.log, ','); if (r) return r; fsopt->new_dev_syntax = false; return 0; } static int ceph_parse_new_source(const char *dev_name, const char *dev_name_end, struct fs_context *fc) { size_t len; struct ceph_fsid fsid; struct ceph_parse_opts_ctx *pctx = fc->fs_private; struct ceph_options *opts = pctx->copts; struct ceph_mount_options *fsopt = pctx->opts; const char *name_start = dev_name; const char *fsid_start, *fs_name_start; if (*dev_name_end != '=') { dout("separator '=' missing in source"); return -EINVAL; } fsid_start = strchr(dev_name, '@'); if (!fsid_start) return invalfc(fc, "missing cluster fsid"); len = fsid_start - name_start; kfree(opts->name); opts->name = kstrndup(name_start, len, GFP_KERNEL); if (!opts->name) return -ENOMEM; dout("using %s entity name", opts->name); ++fsid_start; /* start of cluster fsid */ fs_name_start = strchr(fsid_start, '.'); if (!fs_name_start) return invalfc(fc, "missing file system name"); if (ceph_parse_fsid(fsid_start, &fsid)) return invalfc(fc, "Invalid FSID"); ++fs_name_start; /* start of file system name */ len = dev_name_end - fs_name_start; if (!namespace_equals(fsopt, fs_name_start, len)) return invalfc(fc, "Mismatching mds_namespace"); kfree(fsopt->mds_namespace); fsopt->mds_namespace = kstrndup(fs_name_start, len, GFP_KERNEL); if (!fsopt->mds_namespace) return -ENOMEM; dout("file system (mds namespace) '%s'\n", fsopt->mds_namespace); fsopt->new_dev_syntax = true; return 0; } /* * Parse the source parameter for new device format. Distinguish the device * spec from the path. Try parsing new device format and fallback to old * format if needed. * * New device syntax will looks like: * <device_spec>=/<path> * where * <device_spec> is name@fsid.fsname * <path> is optional, but if present must begin with '/' * (monitor addresses are passed via mount option) * * Old device syntax is: * <server_spec>[,<server_spec>...]:[<path>] * where * <server_spec> is <ip>[:<port>] * <path> is optional, but if present must begin with '/' */ static int ceph_parse_source(struct fs_parameter *param, struct fs_context *fc) { struct ceph_parse_opts_ctx *pctx = fc->fs_private; struct ceph_mount_options *fsopt = pctx->opts; char *dev_name = param->string, *dev_name_end; int ret; dout("'%s'\n", dev_name); if (!dev_name || !*dev_name) return invalfc(fc, "Empty source"); dev_name_end = strchr(dev_name, '/'); if (dev_name_end) { /* * The server_path will include the whole chars from userland * including the leading '/'. */ kfree(fsopt->server_path); fsopt->server_path = kstrdup(dev_name_end, GFP_KERNEL); if (!fsopt->server_path) return -ENOMEM; canonicalize_path(fsopt->server_path); } else { dev_name_end = dev_name + strlen(dev_name); } dev_name_end--; /* back up to separator */ if (dev_name_end < dev_name) return invalfc(fc, "Path missing in source"); dout("device name '%.*s'\n", (int)(dev_name_end - dev_name), dev_name); if (fsopt->server_path) dout("server path '%s'\n", fsopt->server_path); dout("trying new device syntax"); ret = ceph_parse_new_source(dev_name, dev_name_end, fc); if (ret) { if (ret != -EINVAL) return ret; dout("trying old device syntax"); ret = ceph_parse_old_source(dev_name, dev_name_end, fc); if (ret) return ret; } fc->source = param->string; param->string = NULL; return 0; } static int ceph_parse_mon_addr(struct fs_parameter *param, struct fs_context *fc) { struct ceph_parse_opts_ctx *pctx = fc->fs_private; struct ceph_mount_options *fsopt = pctx->opts; kfree(fsopt->mon_addr); fsopt->mon_addr = param->string; param->string = NULL; return ceph_parse_mon_ips(fsopt->mon_addr, strlen(fsopt->mon_addr), pctx->copts, fc->log.log, '/'); } static int ceph_parse_mount_param(struct fs_context *fc, struct fs_parameter *param) { struct ceph_parse_opts_ctx *pctx = fc->fs_private; struct ceph_mount_options *fsopt = pctx->opts; struct fs_parse_result result; unsigned int mode; int token, ret; ret = ceph_parse_param(param, pctx->copts, fc->log.log); if (ret != -ENOPARAM) return ret; token = fs_parse(fc, ceph_mount_parameters, param, &result); dout("%s: fs_parse '%s' token %d\n",__func__, param->key, token); if (token < 0) return token; switch (token) { case Opt_snapdirname: kfree(fsopt->snapdir_name); fsopt->snapdir_name = param->string; param->string = NULL; break; case Opt_mds_namespace: if (!namespace_equals(fsopt, param->string, strlen(param->string))) return invalfc(fc, "Mismatching mds_namespace"); kfree(fsopt->mds_namespace); fsopt->mds_namespace = param->string; param->string = NULL; break; case Opt_recover_session: mode = result.uint_32; if (mode == ceph_recover_session_no) fsopt->flags &= ~CEPH_MOUNT_OPT_CLEANRECOVER; else if (mode == ceph_recover_session_clean) fsopt->flags |= CEPH_MOUNT_OPT_CLEANRECOVER; else BUG(); break; case Opt_source: if (fc->source) return invalfc(fc, "Multiple sources specified"); return ceph_parse_source(param, fc); case Opt_mon_addr: return ceph_parse_mon_addr(param, fc); case Opt_wsize: if (result.uint_32 < PAGE_SIZE || result.uint_32 > CEPH_MAX_WRITE_SIZE) goto out_of_range; fsopt->wsize = ALIGN(result.uint_32, PAGE_SIZE); break; case Opt_rsize: if (result.uint_32 < PAGE_SIZE || result.uint_32 > CEPH_MAX_READ_SIZE) goto out_of_range; fsopt->rsize = ALIGN(result.uint_32, PAGE_SIZE); break; case Opt_rasize: fsopt->rasize = ALIGN(result.uint_32, PAGE_SIZE); break; case Opt_caps_wanted_delay_min: if (result.uint_32 < 1) goto out_of_range; fsopt->caps_wanted_delay_min = result.uint_32; break; case Opt_caps_wanted_delay_max: if (result.uint_32 < 1) goto out_of_range; fsopt->caps_wanted_delay_max = result.uint_32; break; case Opt_caps_max: if (result.int_32 < 0) goto out_of_range; fsopt->caps_max = result.int_32; break; case Opt_readdir_max_entries: if (result.uint_32 < 1) goto out_of_range; fsopt->max_readdir = result.uint_32; break; case Opt_readdir_max_bytes: if (result.uint_32 < PAGE_SIZE && result.uint_32 != 0) goto out_of_range; fsopt->max_readdir_bytes = result.uint_32; break; case Opt_congestion_kb: if (result.uint_32 < 1024) /* at least 1M */ goto out_of_range; fsopt->congestion_kb = result.uint_32; break; case Opt_dirstat: if (!result.negated) fsopt->flags |= CEPH_MOUNT_OPT_DIRSTAT; else fsopt->flags &= ~CEPH_MOUNT_OPT_DIRSTAT; break; case Opt_rbytes: if (!result.negated) fsopt->flags |= CEPH_MOUNT_OPT_RBYTES; else fsopt->flags &= ~CEPH_MOUNT_OPT_RBYTES; break; case Opt_asyncreaddir: if (!result.negated) fsopt->flags &= ~CEPH_MOUNT_OPT_NOASYNCREADDIR; else fsopt->flags |= CEPH_MOUNT_OPT_NOASYNCREADDIR; break; case Opt_dcache: if (!result.negated) fsopt->flags |= CEPH_MOUNT_OPT_DCACHE; else fsopt->flags &= ~CEPH_MOUNT_OPT_DCACHE; break; case Opt_ino32: if (!result.negated) fsopt->flags |= CEPH_MOUNT_OPT_INO32; else fsopt->flags &= ~CEPH_MOUNT_OPT_INO32; break; case Opt_fscache: #ifdef CONFIG_CEPH_FSCACHE kfree(fsopt->fscache_uniq); fsopt->fscache_uniq = NULL; if (result.negated) { fsopt->flags &= ~CEPH_MOUNT_OPT_FSCACHE; } else { fsopt->flags |= CEPH_MOUNT_OPT_FSCACHE; fsopt->fscache_uniq = param->string; param->string = NULL; } break; #else return invalfc(fc, "fscache support is disabled"); #endif case Opt_poolperm: if (!result.negated) fsopt->flags &= ~CEPH_MOUNT_OPT_NOPOOLPERM; else fsopt->flags |= CEPH_MOUNT_OPT_NOPOOLPERM; break; case Opt_require_active_mds: if (!result.negated) fsopt->flags &= ~CEPH_MOUNT_OPT_MOUNTWAIT; else fsopt->flags |= CEPH_MOUNT_OPT_MOUNTWAIT; break; case Opt_quotadf: if (!result.negated) fsopt->flags &= ~CEPH_MOUNT_OPT_NOQUOTADF; else fsopt->flags |= CEPH_MOUNT_OPT_NOQUOTADF; break; case Opt_copyfrom: if (!result.negated) fsopt->flags &= ~CEPH_MOUNT_OPT_NOCOPYFROM; else fsopt->flags |= CEPH_MOUNT_OPT_NOCOPYFROM; break; case Opt_acl: if (!result.negated) { #ifdef CONFIG_CEPH_FS_POSIX_ACL fc->sb_flags |= SB_POSIXACL; #else return invalfc(fc, "POSIX ACL support is disabled"); #endif } else { fc->sb_flags &= ~SB_POSIXACL; } break; case Opt_wsync: if (!result.negated) fsopt->flags &= ~CEPH_MOUNT_OPT_ASYNC_DIROPS; else fsopt->flags |= CEPH_MOUNT_OPT_ASYNC_DIROPS; break; case Opt_pagecache: if (result.negated) fsopt->flags |= CEPH_MOUNT_OPT_NOPAGECACHE; else fsopt->flags &= ~CEPH_MOUNT_OPT_NOPAGECACHE; break; case Opt_sparseread: if (result.negated) fsopt->flags &= ~CEPH_MOUNT_OPT_SPARSEREAD; else fsopt->flags |= CEPH_MOUNT_OPT_SPARSEREAD; break; case Opt_test_dummy_encryption: #ifdef CONFIG_FS_ENCRYPTION fscrypt_free_dummy_policy(&fsopt->dummy_enc_policy); ret = fscrypt_parse_test_dummy_encryption(param, &fsopt->dummy_enc_policy); if (ret == -EINVAL) { warnfc(fc, "Value of option \"%s\" is unrecognized", param->key); } else if (ret == -EEXIST) { warnfc(fc, "Conflicting test_dummy_encryption options"); ret = -EINVAL; } #else warnfc(fc, "FS encryption not supported: test_dummy_encryption mount option ignored"); #endif break; default: BUG(); } return 0; out_of_range: return invalfc(fc, "%s out of range", param->key); } static void destroy_mount_options(struct ceph_mount_options *args) { dout("destroy_mount_options %p\n", args); if (!args) return; kfree(args->snapdir_name); kfree(args->mds_namespace); kfree(args->server_path); kfree(args->fscache_uniq); kfree(args->mon_addr); fscrypt_free_dummy_policy(&args->dummy_enc_policy); kfree(args); } static int strcmp_null(const char *s1, const char *s2) { if (!s1 && !s2) return 0; if (s1 && !s2) return -1; if (!s1 && s2) return 1; return strcmp(s1, s2); } static int compare_mount_options(struct ceph_mount_options *new_fsopt, struct ceph_options *new_opt, struct ceph_fs_client *fsc) { struct ceph_mount_options *fsopt1 = new_fsopt; struct ceph_mount_options *fsopt2 = fsc->mount_options; int ofs = offsetof(struct ceph_mount_options, snapdir_name); int ret; ret = memcmp(fsopt1, fsopt2, ofs); if (ret) return ret; ret = strcmp_null(fsopt1->snapdir_name, fsopt2->snapdir_name); if (ret) return ret; ret = strcmp_null(fsopt1->mds_namespace, fsopt2->mds_namespace); if (ret) return ret; ret = strcmp_null(fsopt1->server_path, fsopt2->server_path); if (ret) return ret; ret = strcmp_null(fsopt1->fscache_uniq, fsopt2->fscache_uniq); if (ret) return ret; ret = strcmp_null(fsopt1->mon_addr, fsopt2->mon_addr); if (ret) return ret; return ceph_compare_options(new_opt, fsc->client); } /** * ceph_show_options - Show mount options in /proc/mounts * @m: seq_file to write to * @root: root of that (sub)tree */ static int ceph_show_options(struct seq_file *m, struct dentry *root) { struct ceph_fs_client *fsc = ceph_sb_to_fs_client(root->d_sb); struct ceph_mount_options *fsopt = fsc->mount_options; size_t pos; int ret; /* a comma between MNT/MS and client options */ seq_putc(m, ','); pos = m->count; ret = ceph_print_client_options(m, fsc->client, false); if (ret) return ret; /* retract our comma if no client options */ if (m->count == pos) m->count--; if (fsopt->flags & CEPH_MOUNT_OPT_DIRSTAT) seq_puts(m, ",dirstat"); if ((fsopt->flags & CEPH_MOUNT_OPT_RBYTES)) seq_puts(m, ",rbytes"); if (fsopt->flags & CEPH_MOUNT_OPT_NOASYNCREADDIR) seq_puts(m, ",noasyncreaddir"); if ((fsopt->flags & CEPH_MOUNT_OPT_DCACHE) == 0) seq_puts(m, ",nodcache"); if (fsopt->flags & CEPH_MOUNT_OPT_INO32) seq_puts(m, ",ino32"); if (fsopt->flags & CEPH_MOUNT_OPT_FSCACHE) { seq_show_option(m, "fsc", fsopt->fscache_uniq); } if (fsopt->flags & CEPH_MOUNT_OPT_NOPOOLPERM) seq_puts(m, ",nopoolperm"); if (fsopt->flags & CEPH_MOUNT_OPT_NOQUOTADF) seq_puts(m, ",noquotadf"); #ifdef CONFIG_CEPH_FS_POSIX_ACL if (root->d_sb->s_flags & SB_POSIXACL) seq_puts(m, ",acl"); else seq_puts(m, ",noacl"); #endif if ((fsopt->flags & CEPH_MOUNT_OPT_NOCOPYFROM) == 0) seq_puts(m, ",copyfrom"); /* dump mds_namespace when old device syntax is in use */ if (fsopt->mds_namespace && !fsopt->new_dev_syntax) seq_show_option(m, "mds_namespace", fsopt->mds_namespace); if (fsopt->mon_addr) seq_printf(m, ",mon_addr=%s", fsopt->mon_addr); if (fsopt->flags & CEPH_MOUNT_OPT_CLEANRECOVER) seq_show_option(m, "recover_session", "clean"); if (!(fsopt->flags & CEPH_MOUNT_OPT_ASYNC_DIROPS)) seq_puts(m, ",wsync"); if (fsopt->flags & CEPH_MOUNT_OPT_NOPAGECACHE) seq_puts(m, ",nopagecache"); if (fsopt->flags & CEPH_MOUNT_OPT_SPARSEREAD) seq_puts(m, ",sparseread"); fscrypt_show_test_dummy_encryption(m, ',', root->d_sb); if (fsopt->wsize != CEPH_MAX_WRITE_SIZE) seq_printf(m, ",wsize=%u", fsopt->wsize); if (fsopt->rsize != CEPH_MAX_READ_SIZE) seq_printf(m, ",rsize=%u", fsopt->rsize); if (fsopt->rasize != CEPH_RASIZE_DEFAULT) seq_printf(m, ",rasize=%u", fsopt->rasize); if (fsopt->congestion_kb != default_congestion_kb()) seq_printf(m, ",write_congestion_kb=%u", fsopt->congestion_kb); if (fsopt->caps_max) seq_printf(m, ",caps_max=%d", fsopt->caps_max); if (fsopt->caps_wanted_delay_min != CEPH_CAPS_WANTED_DELAY_MIN_DEFAULT) seq_printf(m, ",caps_wanted_delay_min=%u", fsopt->caps_wanted_delay_min); if (fsopt->caps_wanted_delay_max != CEPH_CAPS_WANTED_DELAY_MAX_DEFAULT) seq_printf(m, ",caps_wanted_delay_max=%u", fsopt->caps_wanted_delay_max); if (fsopt->max_readdir != CEPH_MAX_READDIR_DEFAULT) seq_printf(m, ",readdir_max_entries=%u", fsopt->max_readdir); if (fsopt->max_readdir_bytes != CEPH_MAX_READDIR_BYTES_DEFAULT) seq_printf(m, ",readdir_max_bytes=%u", fsopt->max_readdir_bytes); if (strcmp(fsopt->snapdir_name, CEPH_SNAPDIRNAME_DEFAULT)) seq_show_option(m, "snapdirname", fsopt->snapdir_name); return 0; } /* * handle any mon messages the standard library doesn't understand. * return error if we don't either. */ static int extra_mon_dispatch(struct ceph_client *client, struct ceph_msg *msg) { struct ceph_fs_client *fsc = client->private; int type = le16_to_cpu(msg->hdr.type); switch (type) { case CEPH_MSG_MDS_MAP: ceph_mdsc_handle_mdsmap(fsc->mdsc, msg); return 0; case CEPH_MSG_FS_MAP_USER: ceph_mdsc_handle_fsmap(fsc->mdsc, msg); return 0; default: return -1; } } /* * create a new fs client * * Success or not, this function consumes @fsopt and @opt. */ static struct ceph_fs_client *create_fs_client(struct ceph_mount_options *fsopt, struct ceph_options *opt) { struct ceph_fs_client *fsc; int err; fsc = kzalloc(sizeof(*fsc), GFP_KERNEL); if (!fsc) { err = -ENOMEM; goto fail; } fsc->client = ceph_create_client(opt, fsc); if (IS_ERR(fsc->client)) { err = PTR_ERR(fsc->client); goto fail; } opt = NULL; /* fsc->client now owns this */ fsc->client->extra_mon_dispatch = extra_mon_dispatch; ceph_set_opt(fsc->client, ABORT_ON_FULL); if (!fsopt->mds_namespace) { ceph_monc_want_map(&fsc->client->monc, CEPH_SUB_MDSMAP, 0, true); } else { ceph_monc_want_map(&fsc->client->monc, CEPH_SUB_FSMAP, 0, false); } fsc->mount_options = fsopt; fsc->sb = NULL; fsc->mount_state = CEPH_MOUNT_MOUNTING; fsc->filp_gen = 1; fsc->have_copy_from2 = true; atomic_long_set(&fsc->writeback_count, 0); fsc->write_congested = false; err = -ENOMEM; /* * The number of concurrent works can be high but they don't need * to be processed in parallel, limit concurrency. */ fsc->inode_wq = alloc_workqueue("ceph-inode", WQ_UNBOUND, 0); if (!fsc->inode_wq) goto fail_client; fsc->cap_wq = alloc_workqueue("ceph-cap", 0, 1); if (!fsc->cap_wq) goto fail_inode_wq; hash_init(fsc->async_unlink_conflict); spin_lock_init(&fsc->async_unlink_conflict_lock); spin_lock(&ceph_fsc_lock); list_add_tail(&fsc->metric_wakeup, &ceph_fsc_list); spin_unlock(&ceph_fsc_lock); return fsc; fail_inode_wq: destroy_workqueue(fsc->inode_wq); fail_client: ceph_destroy_client(fsc->client); fail: kfree(fsc); if (opt) ceph_destroy_options(opt); destroy_mount_options(fsopt); return ERR_PTR(err); } static void flush_fs_workqueues(struct ceph_fs_client *fsc) { flush_workqueue(fsc->inode_wq); flush_workqueue(fsc->cap_wq); } static void destroy_fs_client(struct ceph_fs_client *fsc) { doutc(fsc->client, "%p\n", fsc); spin_lock(&ceph_fsc_lock); list_del(&fsc->metric_wakeup); spin_unlock(&ceph_fsc_lock); ceph_mdsc_destroy(fsc); destroy_workqueue(fsc->inode_wq); destroy_workqueue(fsc->cap_wq); destroy_mount_options(fsc->mount_options); ceph_destroy_client(fsc->client); kfree(fsc); dout("%s: %p done\n", __func__, fsc); } /* * caches */ struct kmem_cache *ceph_inode_cachep; struct kmem_cache *ceph_cap_cachep; struct kmem_cache *ceph_cap_snap_cachep; struct kmem_cache *ceph_cap_flush_cachep; struct kmem_cache *ceph_dentry_cachep; struct kmem_cache *ceph_file_cachep; struct kmem_cache *ceph_dir_file_cachep; struct kmem_cache *ceph_mds_request_cachep; mempool_t *ceph_wb_pagevec_pool; static void ceph_inode_init_once(void *foo) { struct ceph_inode_info *ci = foo; inode_init_once(&ci->netfs.inode); } static int __init init_caches(void) { int error = -ENOMEM; ceph_inode_cachep = kmem_cache_create("ceph_inode_info", sizeof(struct ceph_inode_info), __alignof__(struct ceph_inode_info), SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT, ceph_inode_init_once); if (!ceph_inode_cachep) return -ENOMEM; ceph_cap_cachep = KMEM_CACHE(ceph_cap, 0); if (!ceph_cap_cachep) goto bad_cap; ceph_cap_snap_cachep = KMEM_CACHE(ceph_cap_snap, 0); if (!ceph_cap_snap_cachep) goto bad_cap_snap; ceph_cap_flush_cachep = KMEM_CACHE(ceph_cap_flush, SLAB_RECLAIM_ACCOUNT); if (!ceph_cap_flush_cachep) goto bad_cap_flush; ceph_dentry_cachep = KMEM_CACHE(ceph_dentry_info, SLAB_RECLAIM_ACCOUNT); if (!ceph_dentry_cachep) goto bad_dentry; ceph_file_cachep = KMEM_CACHE(ceph_file_info, 0); if (!ceph_file_cachep) goto bad_file; ceph_dir_file_cachep = KMEM_CACHE(ceph_dir_file_info, 0); if (!ceph_dir_file_cachep) goto bad_dir_file; ceph_mds_request_cachep = KMEM_CACHE(ceph_mds_request, 0); if (!ceph_mds_request_cachep) goto bad_mds_req; ceph_wb_pagevec_pool = mempool_create_kmalloc_pool(10, (CEPH_MAX_WRITE_SIZE >> PAGE_SHIFT) * sizeof(struct page *)); if (!ceph_wb_pagevec_pool) goto bad_pagevec_pool; return 0; bad_pagevec_pool: kmem_cache_destroy(ceph_mds_request_cachep); bad_mds_req: kmem_cache_destroy(ceph_dir_file_cachep); bad_dir_file: kmem_cache_destroy(ceph_file_cachep); bad_file: kmem_cache_destroy(ceph_dentry_cachep); bad_dentry: kmem_cache_destroy(ceph_cap_flush_cachep); bad_cap_flush: kmem_cache_destroy(ceph_cap_snap_cachep); bad_cap_snap: kmem_cache_destroy(ceph_cap_cachep); bad_cap: kmem_cache_destroy(ceph_inode_cachep); return error; } static void destroy_caches(void) { /* * Make sure all delayed rcu free inodes are flushed before we * destroy cache. */ rcu_barrier(); kmem_cache_destroy(ceph_inode_cachep); kmem_cache_destroy(ceph_cap_cachep); kmem_cache_destroy(ceph_cap_snap_cachep); kmem_cache_destroy(ceph_cap_flush_cachep); kmem_cache_destroy(ceph_dentry_cachep); kmem_cache_destroy(ceph_file_cachep); kmem_cache_destroy(ceph_dir_file_cachep); kmem_cache_destroy(ceph_mds_request_cachep); mempool_destroy(ceph_wb_pagevec_pool); } static void __ceph_umount_begin(struct ceph_fs_client *fsc) { ceph_osdc_abort_requests(&fsc->client->osdc, -EIO); ceph_mdsc_force_umount(fsc->mdsc); fsc->filp_gen++; // invalidate open files } /* * ceph_umount_begin - initiate forced umount. Tear down the * mount, skipping steps that may hang while waiting for server(s). */ void ceph_umount_begin(struct super_block *sb) { struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); doutc(fsc->client, "starting forced umount\n"); if (!fsc) return; fsc->mount_state = CEPH_MOUNT_SHUTDOWN; __ceph_umount_begin(fsc); } static const struct super_operations ceph_super_ops = { .alloc_inode = ceph_alloc_inode, .free_inode = ceph_free_inode, .write_inode = ceph_write_inode, .drop_inode = generic_delete_inode, .evict_inode = ceph_evict_inode, .sync_fs = ceph_sync_fs, .put_super = ceph_put_super, .show_options = ceph_show_options, .statfs = ceph_statfs, .umount_begin = ceph_umount_begin, }; /* * Bootstrap mount by opening the root directory. Note the mount * @started time from caller, and time out if this takes too long. */ static struct dentry *open_root_dentry(struct ceph_fs_client *fsc, const char *path, unsigned long started) { struct ceph_client *cl = fsc->client; struct ceph_mds_client *mdsc = fsc->mdsc; struct ceph_mds_request *req = NULL; int err; struct dentry *root; /* open dir */ doutc(cl, "opening '%s'\n", path); req = ceph_mdsc_create_request(mdsc, CEPH_MDS_OP_GETATTR, USE_ANY_MDS); if (IS_ERR(req)) return ERR_CAST(req); req->r_path1 = kstrdup(path, GFP_NOFS); if (!req->r_path1) { root = ERR_PTR(-ENOMEM); goto out; } req->r_ino1.ino = CEPH_INO_ROOT; req->r_ino1.snap = CEPH_NOSNAP; req->r_started = started; req->r_timeout = fsc->client->options->mount_timeout; req->r_args.getattr.mask = cpu_to_le32(CEPH_STAT_CAP_INODE); req->r_num_caps = 2; err = ceph_mdsc_do_request(mdsc, NULL, req); if (err == 0) { struct inode *inode = req->r_target_inode; req->r_target_inode = NULL; doutc(cl, "success\n"); root = d_make_root(inode); if (!root) { root = ERR_PTR(-ENOMEM); goto out; } doutc(cl, "success, root dentry is %p\n", root); } else { root = ERR_PTR(err); } out: ceph_mdsc_put_request(req); return root; } #ifdef CONFIG_FS_ENCRYPTION static int ceph_apply_test_dummy_encryption(struct super_block *sb, struct fs_context *fc, struct ceph_mount_options *fsopt) { struct ceph_fs_client *fsc = sb->s_fs_info; if (!fscrypt_is_dummy_policy_set(&fsopt->dummy_enc_policy)) return 0; /* No changing encryption context on remount. */ if (fc->purpose == FS_CONTEXT_FOR_RECONFIGURE && !fscrypt_is_dummy_policy_set(&fsc->fsc_dummy_enc_policy)) { if (fscrypt_dummy_policies_equal(&fsopt->dummy_enc_policy, &fsc->fsc_dummy_enc_policy)) return 0; errorfc(fc, "Can't set test_dummy_encryption on remount"); return -EINVAL; } /* Also make sure fsopt doesn't contain a conflicting value. */ if (fscrypt_is_dummy_policy_set(&fsc->fsc_dummy_enc_policy)) { if (fscrypt_dummy_policies_equal(&fsopt->dummy_enc_policy, &fsc->fsc_dummy_enc_policy)) return 0; errorfc(fc, "Conflicting test_dummy_encryption options"); return -EINVAL; } fsc->fsc_dummy_enc_policy = fsopt->dummy_enc_policy; memset(&fsopt->dummy_enc_policy, 0, sizeof(fsopt->dummy_enc_policy)); warnfc(fc, "test_dummy_encryption mode enabled"); return 0; } #else static int ceph_apply_test_dummy_encryption(struct super_block *sb, struct fs_context *fc, struct ceph_mount_options *fsopt) { return 0; } #endif /* * mount: join the ceph cluster, and open root directory. */ static struct dentry *ceph_real_mount(struct ceph_fs_client *fsc, struct fs_context *fc) { struct ceph_client *cl = fsc->client; int err; unsigned long started = jiffies; /* note the start time */ struct dentry *root; doutc(cl, "mount start %p\n", fsc); mutex_lock(&fsc->client->mount_mutex); if (!fsc->sb->s_root) { const char *path = fsc->mount_options->server_path ? fsc->mount_options->server_path + 1 : ""; err = __ceph_open_session(fsc->client, started); if (err < 0) goto out; /* setup fscache */ if (fsc->mount_options->flags & CEPH_MOUNT_OPT_FSCACHE) { err = ceph_fscache_register_fs(fsc, fc); if (err < 0) goto out; } err = ceph_apply_test_dummy_encryption(fsc->sb, fc, fsc->mount_options); if (err) goto out; doutc(cl, "mount opening path '%s'\n", path); ceph_fs_debugfs_init(fsc); root = open_root_dentry(fsc, path, started); if (IS_ERR(root)) { err = PTR_ERR(root); goto out; } fsc->sb->s_root = dget(root); } else { root = dget(fsc->sb->s_root); } fsc->mount_state = CEPH_MOUNT_MOUNTED; doutc(cl, "mount success\n"); mutex_unlock(&fsc->client->mount_mutex); return root; out: mutex_unlock(&fsc->client->mount_mutex); ceph_fscrypt_free_dummy_policy(fsc); return ERR_PTR(err); } static int ceph_set_super(struct super_block *s, struct fs_context *fc) { struct ceph_fs_client *fsc = s->s_fs_info; struct ceph_client *cl = fsc->client; int ret; doutc(cl, "%p\n", s); s->s_maxbytes = MAX_LFS_FILESIZE; s->s_xattr = ceph_xattr_handlers; fsc->sb = s; fsc->max_file_size = 1ULL << 40; /* temp value until we get mdsmap */ s->s_op = &ceph_super_ops; s->s_d_op = &ceph_dentry_ops; s->s_export_op = &ceph_export_ops; s->s_time_gran = 1; s->s_time_min = 0; s->s_time_max = U32_MAX; s->s_flags |= SB_NODIRATIME | SB_NOATIME; ceph_fscrypt_set_ops(s); ret = set_anon_super_fc(s, fc); if (ret != 0) fsc->sb = NULL; return ret; } /* * share superblock if same fs AND options */ static int ceph_compare_super(struct super_block *sb, struct fs_context *fc) { struct ceph_fs_client *new = fc->s_fs_info; struct ceph_mount_options *fsopt = new->mount_options; struct ceph_options *opt = new->client->options; struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); struct ceph_client *cl = fsc->client; doutc(cl, "%p\n", sb); if (compare_mount_options(fsopt, opt, fsc)) { doutc(cl, "monitor(s)/mount options don't match\n"); return 0; } if ((opt->flags & CEPH_OPT_FSID) && ceph_fsid_compare(&opt->fsid, &fsc->client->fsid)) { doutc(cl, "fsid doesn't match\n"); return 0; } if (fc->sb_flags != (sb->s_flags & ~SB_BORN)) { doutc(cl, "flags differ\n"); return 0; } if (fsc->blocklisted && !ceph_test_mount_opt(fsc, CLEANRECOVER)) { doutc(cl, "client is blocklisted (and CLEANRECOVER is not set)\n"); return 0; } if (fsc->mount_state == CEPH_MOUNT_SHUTDOWN) { doutc(cl, "client has been forcibly unmounted\n"); return 0; } return 1; } /* * construct our own bdi so we can control readahead, etc. */ static atomic_long_t bdi_seq = ATOMIC_LONG_INIT(0); static int ceph_setup_bdi(struct super_block *sb, struct ceph_fs_client *fsc) { int err; err = super_setup_bdi_name(sb, "ceph-%ld", atomic_long_inc_return(&bdi_seq)); if (err) return err; /* set ra_pages based on rasize mount option? */ sb->s_bdi->ra_pages = fsc->mount_options->rasize >> PAGE_SHIFT; /* set io_pages based on max osd read size */ sb->s_bdi->io_pages = fsc->mount_options->rsize >> PAGE_SHIFT; return 0; } static int ceph_get_tree(struct fs_context *fc) { struct ceph_parse_opts_ctx *pctx = fc->fs_private; struct ceph_mount_options *fsopt = pctx->opts; struct super_block *sb; struct ceph_fs_client *fsc; struct dentry *res; int (*compare_super)(struct super_block *, struct fs_context *) = ceph_compare_super; int err; dout("ceph_get_tree\n"); if (!fc->source) return invalfc(fc, "No source"); if (fsopt->new_dev_syntax && !fsopt->mon_addr) return invalfc(fc, "No monitor address"); /* create client (which we may/may not use) */ fsc = create_fs_client(pctx->opts, pctx->copts); pctx->opts = NULL; pctx->copts = NULL; if (IS_ERR(fsc)) { err = PTR_ERR(fsc); goto out_final; } err = ceph_mdsc_init(fsc); if (err < 0) goto out; if (ceph_test_opt(fsc->client, NOSHARE)) compare_super = NULL; fc->s_fs_info = fsc; sb = sget_fc(fc, compare_super, ceph_set_super); fc->s_fs_info = NULL; if (IS_ERR(sb)) { err = PTR_ERR(sb); goto out; } if (ceph_sb_to_fs_client(sb) != fsc) { destroy_fs_client(fsc); fsc = ceph_sb_to_fs_client(sb); dout("get_sb got existing client %p\n", fsc); } else { dout("get_sb using new client %p\n", fsc); err = ceph_setup_bdi(sb, fsc); if (err < 0) goto out_splat; } res = ceph_real_mount(fsc, fc); if (IS_ERR(res)) { err = PTR_ERR(res); goto out_splat; } doutc(fsc->client, "root %p inode %p ino %llx.%llx\n", res, d_inode(res), ceph_vinop(d_inode(res))); fc->root = fsc->sb->s_root; return 0; out_splat: if (!ceph_mdsmap_is_cluster_available(fsc->mdsc->mdsmap)) { pr_info("No mds server is up or the cluster is laggy\n"); err = -EHOSTUNREACH; } ceph_mdsc_close_sessions(fsc->mdsc); deactivate_locked_super(sb); goto out_final; out: destroy_fs_client(fsc); out_final: dout("ceph_get_tree fail %d\n", err); return err; } static void ceph_free_fc(struct fs_context *fc) { struct ceph_parse_opts_ctx *pctx = fc->fs_private; if (pctx) { destroy_mount_options(pctx->opts); ceph_destroy_options(pctx->copts); kfree(pctx); } } static int ceph_reconfigure_fc(struct fs_context *fc) { int err; struct ceph_parse_opts_ctx *pctx = fc->fs_private; struct ceph_mount_options *fsopt = pctx->opts; struct super_block *sb = fc->root->d_sb; struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); err = ceph_apply_test_dummy_encryption(sb, fc, fsopt); if (err) return err; if (fsopt->flags & CEPH_MOUNT_OPT_ASYNC_DIROPS) ceph_set_mount_opt(fsc, ASYNC_DIROPS); else ceph_clear_mount_opt(fsc, ASYNC_DIROPS); if (fsopt->flags & CEPH_MOUNT_OPT_SPARSEREAD) ceph_set_mount_opt(fsc, SPARSEREAD); else ceph_clear_mount_opt(fsc, SPARSEREAD); if (strcmp_null(fsc->mount_options->mon_addr, fsopt->mon_addr)) { kfree(fsc->mount_options->mon_addr); fsc->mount_options->mon_addr = fsopt->mon_addr; fsopt->mon_addr = NULL; pr_notice_client(fsc->client, "monitor addresses recorded, but not used for reconnection"); } sync_filesystem(sb); return 0; } static const struct fs_context_operations ceph_context_ops = { .free = ceph_free_fc, .parse_param = ceph_parse_mount_param, .get_tree = ceph_get_tree, .reconfigure = ceph_reconfigure_fc, }; /* * Set up the filesystem mount context. */ static int ceph_init_fs_context(struct fs_context *fc) { struct ceph_parse_opts_ctx *pctx; struct ceph_mount_options *fsopt; pctx = kzalloc(sizeof(*pctx), GFP_KERNEL); if (!pctx) return -ENOMEM; pctx->copts = ceph_alloc_options(); if (!pctx->copts) goto nomem; pctx->opts = kzalloc(sizeof(*pctx->opts), GFP_KERNEL); if (!pctx->opts) goto nomem; fsopt = pctx->opts; fsopt->flags = CEPH_MOUNT_OPT_DEFAULT; fsopt->wsize = CEPH_MAX_WRITE_SIZE; fsopt->rsize = CEPH_MAX_READ_SIZE; fsopt->rasize = CEPH_RASIZE_DEFAULT; fsopt->snapdir_name = kstrdup(CEPH_SNAPDIRNAME_DEFAULT, GFP_KERNEL); if (!fsopt->snapdir_name) goto nomem; fsopt->caps_wanted_delay_min = CEPH_CAPS_WANTED_DELAY_MIN_DEFAULT; fsopt->caps_wanted_delay_max = CEPH_CAPS_WANTED_DELAY_MAX_DEFAULT; fsopt->max_readdir = CEPH_MAX_READDIR_DEFAULT; fsopt->max_readdir_bytes = CEPH_MAX_READDIR_BYTES_DEFAULT; fsopt->congestion_kb = default_congestion_kb(); #ifdef CONFIG_CEPH_FS_POSIX_ACL fc->sb_flags |= SB_POSIXACL; #endif fc->fs_private = pctx; fc->ops = &ceph_context_ops; return 0; nomem: destroy_mount_options(pctx->opts); ceph_destroy_options(pctx->copts); kfree(pctx); return -ENOMEM; } /* * Return true if it successfully increases the blocker counter, * or false if the mdsc is in stopping and flushed state. */ static bool __inc_stopping_blocker(struct ceph_mds_client *mdsc) { spin_lock(&mdsc->stopping_lock); if (mdsc->stopping >= CEPH_MDSC_STOPPING_FLUSHING) { spin_unlock(&mdsc->stopping_lock); return false; } atomic_inc(&mdsc->stopping_blockers); spin_unlock(&mdsc->stopping_lock); return true; } static void __dec_stopping_blocker(struct ceph_mds_client *mdsc) { spin_lock(&mdsc->stopping_lock); if (!atomic_dec_return(&mdsc->stopping_blockers) && mdsc->stopping >= CEPH_MDSC_STOPPING_FLUSHING) complete_all(&mdsc->stopping_waiter); spin_unlock(&mdsc->stopping_lock); } /* For metadata IO requests */ bool ceph_inc_mds_stopping_blocker(struct ceph_mds_client *mdsc, struct ceph_mds_session *session) { mutex_lock(&session->s_mutex); inc_session_sequence(session); mutex_unlock(&session->s_mutex); return __inc_stopping_blocker(mdsc); } void ceph_dec_mds_stopping_blocker(struct ceph_mds_client *mdsc) { __dec_stopping_blocker(mdsc); } /* For data IO requests */ bool ceph_inc_osd_stopping_blocker(struct ceph_mds_client *mdsc) { return __inc_stopping_blocker(mdsc); } void ceph_dec_osd_stopping_blocker(struct ceph_mds_client *mdsc) { __dec_stopping_blocker(mdsc); } static void ceph_kill_sb(struct super_block *s) { struct ceph_fs_client *fsc = ceph_sb_to_fs_client(s); struct ceph_client *cl = fsc->client; struct ceph_mds_client *mdsc = fsc->mdsc; bool wait; doutc(cl, "%p\n", s); ceph_mdsc_pre_umount(mdsc); flush_fs_workqueues(fsc); /* * Though the kill_anon_super() will finally trigger the * sync_filesystem() anyway, we still need to do it here and * then bump the stage of shutdown. This will allow us to * drop any further message, which will increase the inodes' * i_count reference counters but makes no sense any more, * from MDSs. * * Without this when evicting the inodes it may fail in the * kill_anon_super(), which will trigger a warning when * destroying the fscrypt keyring and then possibly trigger * a further crash in ceph module when the iput() tries to * evict the inodes later. */ sync_filesystem(s); spin_lock(&mdsc->stopping_lock); mdsc->stopping = CEPH_MDSC_STOPPING_FLUSHING; wait = !!atomic_read(&mdsc->stopping_blockers); spin_unlock(&mdsc->stopping_lock); if (wait && atomic_read(&mdsc->stopping_blockers)) { long timeleft = wait_for_completion_killable_timeout( &mdsc->stopping_waiter, fsc->client->options->mount_timeout); if (!timeleft) /* timed out */ pr_warn_client(cl, "umount timed out, %ld\n", timeleft); else if (timeleft < 0) /* killed */ pr_warn_client(cl, "umount was killed, %ld\n", timeleft); } mdsc->stopping = CEPH_MDSC_STOPPING_FLUSHED; kill_anon_super(s); fsc->client->extra_mon_dispatch = NULL; ceph_fs_debugfs_cleanup(fsc); ceph_fscache_unregister_fs(fsc); destroy_fs_client(fsc); } static struct file_system_type ceph_fs_type = { .owner = THIS_MODULE, .name = "ceph", .init_fs_context = ceph_init_fs_context, .kill_sb = ceph_kill_sb, .fs_flags = FS_RENAME_DOES_D_MOVE | FS_ALLOW_IDMAP, }; MODULE_ALIAS_FS("ceph"); int ceph_force_reconnect(struct super_block *sb) { struct ceph_fs_client *fsc = ceph_sb_to_fs_client(sb); int err = 0; fsc->mount_state = CEPH_MOUNT_RECOVER; __ceph_umount_begin(fsc); /* Make sure all page caches get invalidated. * see remove_session_caps_cb() */ flush_workqueue(fsc->inode_wq); /* In case that we were blocklisted. This also reset * all mon/osd connections */ ceph_reset_client_addr(fsc->client); ceph_osdc_clear_abort_err(&fsc->client->osdc); fsc->blocklisted = false; fsc->mount_state = CEPH_MOUNT_MOUNTED; if (sb->s_root) { err = __ceph_do_getattr(d_inode(sb->s_root), NULL, CEPH_STAT_CAP_INODE, true); } return err; } static int __init init_ceph(void) { int ret = init_caches(); if (ret) goto out; ceph_flock_init(); ret = register_filesystem(&ceph_fs_type); if (ret) goto out_caches; pr_info("loaded (mds proto %d)\n", CEPH_MDSC_PROTOCOL); return 0; out_caches: destroy_caches(); out: return ret; } static void __exit exit_ceph(void) { dout("exit_ceph\n"); unregister_filesystem(&ceph_fs_type); destroy_caches(); } static int param_set_metrics(const char *val, const struct kernel_param *kp) { struct ceph_fs_client *fsc; int ret; ret = param_set_bool(val, kp); if (ret) { pr_err("Failed to parse sending metrics switch value '%s'\n", val); return ret; } else if (!disable_send_metrics) { // wake up all the mds clients spin_lock(&ceph_fsc_lock); list_for_each_entry(fsc, &ceph_fsc_list, metric_wakeup) { metric_schedule_delayed(&fsc->mdsc->metric); } spin_unlock(&ceph_fsc_lock); } return 0; } static const struct kernel_param_ops param_ops_metrics = { .set = param_set_metrics, .get = param_get_bool, }; bool disable_send_metrics = false; module_param_cb(disable_send_metrics, ¶m_ops_metrics, &disable_send_metrics, 0644); MODULE_PARM_DESC(disable_send_metrics, "Enable sending perf metrics to ceph cluster (default: on)"); /* for both v1 and v2 syntax */ static bool mount_support = true; static const struct kernel_param_ops param_ops_mount_syntax = { .get = param_get_bool, }; module_param_cb(mount_syntax_v1, ¶m_ops_mount_syntax, &mount_support, 0444); module_param_cb(mount_syntax_v2, ¶m_ops_mount_syntax, &mount_support, 0444); bool enable_unsafe_idmap = false; module_param(enable_unsafe_idmap, bool, 0644); MODULE_PARM_DESC(enable_unsafe_idmap, "Allow to use idmapped mounts with MDS without CEPHFS_FEATURE_HAS_OWNER_UIDGID"); module_init(init_ceph); module_exit(exit_ceph); MODULE_AUTHOR("Sage Weil <sage@newdream.net>"); MODULE_AUTHOR("Yehuda Sadeh <yehuda@hq.newdream.net>"); MODULE_AUTHOR("Patience Warnick <patience@newdream.net>"); MODULE_DESCRIPTION("Ceph filesystem for Linux"); MODULE_LICENSE("GPL"); |
| 23 13 21 21 12 23 33 33 23 23 23 23 32 28 9 33 33 33 5 28 28 9 33 33 28 9 33 12 24 14 23 33 33 33 23 23 2 22 2 2 2 2 2 36 36 34 3 3 35 35 1 5 5 5 5 5 1 1 2 2 2 2 2 9 1 8 2 2 2 4 2 4 2 2 2 2 4 2 2 2 2 4 4 1 4 3 4 1 4 1 1 1 4 1 4 3 2 4 4 1 2 2 4 1 1 1 1 1 1 1 1 1 1 1 1 1 1 4 4 4 1 4 4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 | // SPDX-License-Identifier: GPL-2.0 /* * fs/f2fs/namei.c * * Copyright (c) 2012 Samsung Electronics Co., Ltd. * http://www.samsung.com/ */ #include <linux/fs.h> #include <linux/f2fs_fs.h> #include <linux/pagemap.h> #include <linux/sched.h> #include <linux/ctype.h> #include <linux/random.h> #include <linux/dcache.h> #include <linux/namei.h> #include <linux/quotaops.h> #include "f2fs.h" #include "node.h" #include "segment.h" #include "xattr.h" #include "acl.h" #include <trace/events/f2fs.h> static inline bool is_extension_exist(const unsigned char *s, const char *sub, bool tmp_ext, bool tmp_dot) { size_t slen = strlen(s); size_t sublen = strlen(sub); int i; if (sublen == 1 && *sub == '*') return true; /* * filename format of multimedia file should be defined as: * "filename + '.' + extension + (optional: '.' + temp extension)". */ if (slen < sublen + 2) return false; if (!tmp_ext) { /* file has no temp extension */ if (s[slen - sublen - 1] != '.') return false; return !strncasecmp(s + slen - sublen, sub, sublen); } for (i = 1; i < slen - sublen; i++) { if (s[i] != '.') continue; if (!strncasecmp(s + i + 1, sub, sublen)) { if (!tmp_dot) return true; if (i == slen - sublen - 1 || s[i + 1 + sublen] == '.') return true; } } return false; } static inline bool is_temperature_extension(const unsigned char *s, const char *sub) { return is_extension_exist(s, sub, true, false); } static inline bool is_compress_extension(const unsigned char *s, const char *sub) { return is_extension_exist(s, sub, true, true); } int f2fs_update_extension_list(struct f2fs_sb_info *sbi, const char *name, bool hot, bool set) { __u8 (*extlist)[F2FS_EXTENSION_LEN] = sbi->raw_super->extension_list; int cold_count = le32_to_cpu(sbi->raw_super->extension_count); int hot_count = sbi->raw_super->hot_ext_count; int total_count = cold_count + hot_count; int start, count; int i; if (set) { if (total_count == F2FS_MAX_EXTENSION) return -EINVAL; } else { if (!hot && !cold_count) return -EINVAL; if (hot && !hot_count) return -EINVAL; } if (hot) { start = cold_count; count = total_count; } else { start = 0; count = cold_count; } for (i = start; i < count; i++) { if (strcmp(name, extlist[i])) continue; if (set) return -EINVAL; memcpy(extlist[i], extlist[i + 1], F2FS_EXTENSION_LEN * (total_count - i - 1)); memset(extlist[total_count - 1], 0, F2FS_EXTENSION_LEN); if (hot) sbi->raw_super->hot_ext_count = hot_count - 1; else sbi->raw_super->extension_count = cpu_to_le32(cold_count - 1); return 0; } if (!set) return -EINVAL; if (hot) { memcpy(extlist[count], name, strlen(name)); sbi->raw_super->hot_ext_count = hot_count + 1; } else { char buf[F2FS_MAX_EXTENSION][F2FS_EXTENSION_LEN]; memcpy(buf, &extlist[cold_count], F2FS_EXTENSION_LEN * hot_count); memset(extlist[cold_count], 0, F2FS_EXTENSION_LEN); memcpy(extlist[cold_count], name, strlen(name)); memcpy(&extlist[cold_count + 1], buf, F2FS_EXTENSION_LEN * hot_count); sbi->raw_super->extension_count = cpu_to_le32(cold_count + 1); } return 0; } static void set_compress_new_inode(struct f2fs_sb_info *sbi, struct inode *dir, struct inode *inode, const unsigned char *name) { __u8 (*extlist)[F2FS_EXTENSION_LEN] = sbi->raw_super->extension_list; unsigned char (*noext)[F2FS_EXTENSION_LEN] = F2FS_OPTION(sbi).noextensions; unsigned char (*ext)[F2FS_EXTENSION_LEN] = F2FS_OPTION(sbi).extensions; unsigned char ext_cnt = F2FS_OPTION(sbi).compress_ext_cnt; unsigned char noext_cnt = F2FS_OPTION(sbi).nocompress_ext_cnt; int i, cold_count, hot_count; if (!f2fs_sb_has_compression(sbi)) return; if (S_ISDIR(inode->i_mode)) goto inherit_comp; /* This name comes only from normal files. */ if (!name) return; /* Don't compress hot files. */ f2fs_down_read(&sbi->sb_lock); cold_count = le32_to_cpu(sbi->raw_super->extension_count); hot_count = sbi->raw_super->hot_ext_count; for (i = cold_count; i < cold_count + hot_count; i++) if (is_temperature_extension(name, extlist[i])) break; f2fs_up_read(&sbi->sb_lock); if (i < (cold_count + hot_count)) return; /* Don't compress unallowed extension. */ for (i = 0; i < noext_cnt; i++) if (is_compress_extension(name, noext[i])) return; /* Compress wanting extension. */ for (i = 0; i < ext_cnt; i++) { if (is_compress_extension(name, ext[i])) { set_compress_context(inode); return; } } inherit_comp: /* Inherit the {no-}compression flag in directory */ if (F2FS_I(dir)->i_flags & F2FS_NOCOMP_FL) { F2FS_I(inode)->i_flags |= F2FS_NOCOMP_FL; f2fs_mark_inode_dirty_sync(inode, true); } else if (F2FS_I(dir)->i_flags & F2FS_COMPR_FL) { set_compress_context(inode); } } /* * Set file's temperature for hot/cold data separation */ static void set_file_temperature(struct f2fs_sb_info *sbi, struct inode *inode, const unsigned char *name) { __u8 (*extlist)[F2FS_EXTENSION_LEN] = sbi->raw_super->extension_list; int i, cold_count, hot_count; f2fs_down_read(&sbi->sb_lock); cold_count = le32_to_cpu(sbi->raw_super->extension_count); hot_count = sbi->raw_super->hot_ext_count; for (i = 0; i < cold_count + hot_count; i++) if (is_temperature_extension(name, extlist[i])) break; f2fs_up_read(&sbi->sb_lock); if (i == cold_count + hot_count) return; if (i < cold_count) file_set_cold(inode); else file_set_hot(inode); } static struct inode *f2fs_new_inode(struct mnt_idmap *idmap, struct inode *dir, umode_t mode, const char *name) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct f2fs_inode_info *fi; nid_t ino; struct inode *inode; bool nid_free = false; bool encrypt = false; int xattr_size = 0; int err; inode = new_inode(dir->i_sb); if (!inode) return ERR_PTR(-ENOMEM); if (!f2fs_alloc_nid(sbi, &ino)) { err = -ENOSPC; goto fail; } nid_free = true; inode_init_owner(idmap, inode, dir, mode); fi = F2FS_I(inode); inode->i_ino = ino; inode->i_blocks = 0; simple_inode_init_ts(inode); fi->i_crtime = inode_get_mtime(inode); inode->i_generation = get_random_u32(); if (S_ISDIR(inode->i_mode)) fi->i_current_depth = 1; err = insert_inode_locked(inode); if (err) { err = -EINVAL; goto fail; } if (f2fs_sb_has_project_quota(sbi) && (F2FS_I(dir)->i_flags & F2FS_PROJINHERIT_FL)) fi->i_projid = F2FS_I(dir)->i_projid; else fi->i_projid = make_kprojid(&init_user_ns, F2FS_DEF_PROJID); err = fscrypt_prepare_new_inode(dir, inode, &encrypt); if (err) goto fail_drop; err = f2fs_dquot_initialize(inode); if (err) goto fail_drop; set_inode_flag(inode, FI_NEW_INODE); if (encrypt) f2fs_set_encrypted_inode(inode); if (f2fs_sb_has_extra_attr(sbi)) { set_inode_flag(inode, FI_EXTRA_ATTR); fi->i_extra_isize = F2FS_TOTAL_EXTRA_ATTR_SIZE; } if (test_opt(sbi, INLINE_XATTR)) set_inode_flag(inode, FI_INLINE_XATTR); if (f2fs_may_inline_dentry(inode)) set_inode_flag(inode, FI_INLINE_DENTRY); if (f2fs_sb_has_flexible_inline_xattr(sbi)) { f2fs_bug_on(sbi, !f2fs_has_extra_attr(inode)); if (f2fs_has_inline_xattr(inode)) xattr_size = F2FS_OPTION(sbi).inline_xattr_size; /* Otherwise, will be 0 */ } else if (f2fs_has_inline_xattr(inode) || f2fs_has_inline_dentry(inode)) { xattr_size = DEFAULT_INLINE_XATTR_ADDRS; } fi->i_inline_xattr_size = xattr_size; fi->i_flags = f2fs_mask_flags(mode, F2FS_I(dir)->i_flags & F2FS_FL_INHERITED); if (S_ISDIR(inode->i_mode)) fi->i_flags |= F2FS_INDEX_FL; if (fi->i_flags & F2FS_PROJINHERIT_FL) set_inode_flag(inode, FI_PROJ_INHERIT); /* Check compression first. */ set_compress_new_inode(sbi, dir, inode, name); /* Should enable inline_data after compression set */ if (test_opt(sbi, INLINE_DATA) && f2fs_may_inline_data(inode)) set_inode_flag(inode, FI_INLINE_DATA); if (name && !test_opt(sbi, DISABLE_EXT_IDENTIFY)) set_file_temperature(sbi, inode, name); stat_inc_inline_xattr(inode); stat_inc_inline_inode(inode); stat_inc_inline_dir(inode); f2fs_set_inode_flags(inode); f2fs_init_extent_tree(inode); trace_f2fs_new_inode(inode, 0); return inode; fail: trace_f2fs_new_inode(inode, err); make_bad_inode(inode); if (nid_free) set_inode_flag(inode, FI_FREE_NID); iput(inode); return ERR_PTR(err); fail_drop: trace_f2fs_new_inode(inode, err); dquot_drop(inode); inode->i_flags |= S_NOQUOTA; if (nid_free) set_inode_flag(inode, FI_FREE_NID); clear_nlink(inode); unlock_new_inode(inode); iput(inode); return ERR_PTR(err); } static int f2fs_create(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct inode *inode; nid_t ino = 0; int err; if (unlikely(f2fs_cp_error(sbi))) return -EIO; if (!f2fs_is_checkpoint_ready(sbi)) return -ENOSPC; err = f2fs_dquot_initialize(dir); if (err) return err; inode = f2fs_new_inode(idmap, dir, mode, dentry->d_name.name); if (IS_ERR(inode)) return PTR_ERR(inode); inode->i_op = &f2fs_file_inode_operations; inode->i_fop = &f2fs_file_operations; inode->i_mapping->a_ops = &f2fs_dblock_aops; ino = inode->i_ino; f2fs_lock_op(sbi); err = f2fs_add_link(dentry, inode); if (err) goto out; f2fs_unlock_op(sbi); f2fs_alloc_nid_done(sbi, ino); d_instantiate_new(dentry, inode); if (IS_DIRSYNC(dir)) f2fs_sync_fs(sbi->sb, 1); f2fs_balance_fs(sbi, true); return 0; out: f2fs_handle_failed_inode(inode); return err; } static int f2fs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) { struct inode *inode = d_inode(old_dentry); struct f2fs_sb_info *sbi = F2FS_I_SB(dir); int err; if (unlikely(f2fs_cp_error(sbi))) return -EIO; if (!f2fs_is_checkpoint_ready(sbi)) return -ENOSPC; err = fscrypt_prepare_link(old_dentry, dir, dentry); if (err) return err; if (is_inode_flag_set(dir, FI_PROJ_INHERIT) && (!projid_eq(F2FS_I(dir)->i_projid, F2FS_I(old_dentry->d_inode)->i_projid))) return -EXDEV; err = f2fs_dquot_initialize(dir); if (err) return err; f2fs_balance_fs(sbi, true); inode_set_ctime_current(inode); ihold(inode); set_inode_flag(inode, FI_INC_LINK); f2fs_lock_op(sbi); err = f2fs_add_link(dentry, inode); if (err) goto out; f2fs_unlock_op(sbi); d_instantiate(dentry, inode); if (IS_DIRSYNC(dir)) f2fs_sync_fs(sbi->sb, 1); return 0; out: clear_inode_flag(inode, FI_INC_LINK); iput(inode); f2fs_unlock_op(sbi); return err; } struct dentry *f2fs_get_parent(struct dentry *child) { struct page *page; unsigned long ino = f2fs_inode_by_name(d_inode(child), &dotdot_name, &page); if (!ino) { if (IS_ERR(page)) return ERR_CAST(page); return ERR_PTR(-ENOENT); } return d_obtain_alias(f2fs_iget(child->d_sb, ino)); } static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) { struct inode *inode = NULL; struct f2fs_dir_entry *de; struct page *page; struct dentry *new; nid_t ino = -1; int err = 0; struct f2fs_filename fname; trace_f2fs_lookup_start(dir, dentry, flags); if (dentry->d_name.len > F2FS_NAME_LEN) { err = -ENAMETOOLONG; goto out; } err = f2fs_prepare_lookup(dir, dentry, &fname); if (err == -ENOENT) goto out_splice; if (err) goto out; de = __f2fs_find_entry(dir, &fname, &page); f2fs_free_filename(&fname); if (!de) { if (IS_ERR(page)) { err = PTR_ERR(page); goto out; } err = -ENOENT; goto out_splice; } ino = le32_to_cpu(de->ino); f2fs_put_page(page, 0); inode = f2fs_iget(dir->i_sb, ino); if (IS_ERR(inode)) { err = PTR_ERR(inode); goto out; } if (IS_ENCRYPTED(dir) && (S_ISDIR(inode->i_mode) || S_ISLNK(inode->i_mode)) && !fscrypt_has_permitted_context(dir, inode)) { f2fs_warn(F2FS_I_SB(inode), "Inconsistent encryption contexts: %lu/%lu", dir->i_ino, inode->i_ino); err = -EPERM; goto out_iput; } out_splice: if (IS_ENABLED(CONFIG_UNICODE) && !inode && IS_CASEFOLDED(dir)) { /* Eventually we want to call d_add_ci(dentry, NULL) * for negative dentries in the encoding case as * well. For now, prevent the negative dentry * from being cached. */ trace_f2fs_lookup_end(dir, dentry, ino, err); return NULL; } new = d_splice_alias(inode, dentry); trace_f2fs_lookup_end(dir, !IS_ERR_OR_NULL(new) ? new : dentry, ino, IS_ERR(new) ? PTR_ERR(new) : err); return new; out_iput: iput(inode); out: trace_f2fs_lookup_end(dir, dentry, ino, err); return ERR_PTR(err); } static int f2fs_unlink(struct inode *dir, struct dentry *dentry) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct inode *inode = d_inode(dentry); struct f2fs_dir_entry *de; struct page *page; int err; trace_f2fs_unlink_enter(dir, dentry); if (unlikely(f2fs_cp_error(sbi))) { err = -EIO; goto fail; } err = f2fs_dquot_initialize(dir); if (err) goto fail; err = f2fs_dquot_initialize(inode); if (err) goto fail; de = f2fs_find_entry(dir, &dentry->d_name, &page); if (!de) { if (IS_ERR(page)) err = PTR_ERR(page); goto fail; } f2fs_balance_fs(sbi, true); f2fs_lock_op(sbi); err = f2fs_acquire_orphan_inode(sbi); if (err) { f2fs_unlock_op(sbi); f2fs_put_page(page, 0); goto fail; } f2fs_delete_entry(de, page, dir, inode); f2fs_unlock_op(sbi); /* VFS negative dentries are incompatible with Encoding and * Case-insensitiveness. Eventually we'll want avoid * invalidating the dentries here, alongside with returning the * negative dentries at f2fs_lookup(), when it is better * supported by the VFS for the CI case. */ if (IS_ENABLED(CONFIG_UNICODE) && IS_CASEFOLDED(dir)) d_invalidate(dentry); if (IS_DIRSYNC(dir)) f2fs_sync_fs(sbi->sb, 1); fail: trace_f2fs_unlink_exit(inode, err); return err; } static const char *f2fs_get_link(struct dentry *dentry, struct inode *inode, struct delayed_call *done) { const char *link = page_get_link(dentry, inode, done); if (!IS_ERR(link) && !*link) { /* this is broken symlink case */ do_delayed_call(done); clear_delayed_call(done); link = ERR_PTR(-ENOENT); } return link; } static int f2fs_symlink(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, const char *symname) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct inode *inode; size_t len = strlen(symname); struct fscrypt_str disk_link; int err; if (unlikely(f2fs_cp_error(sbi))) return -EIO; if (!f2fs_is_checkpoint_ready(sbi)) return -ENOSPC; err = fscrypt_prepare_symlink(dir, symname, len, dir->i_sb->s_blocksize, &disk_link); if (err) return err; err = f2fs_dquot_initialize(dir); if (err) return err; inode = f2fs_new_inode(idmap, dir, S_IFLNK | S_IRWXUGO, NULL); if (IS_ERR(inode)) return PTR_ERR(inode); if (IS_ENCRYPTED(inode)) inode->i_op = &f2fs_encrypted_symlink_inode_operations; else inode->i_op = &f2fs_symlink_inode_operations; inode_nohighmem(inode); inode->i_mapping->a_ops = &f2fs_dblock_aops; f2fs_lock_op(sbi); err = f2fs_add_link(dentry, inode); if (err) goto out_f2fs_handle_failed_inode; f2fs_unlock_op(sbi); f2fs_alloc_nid_done(sbi, inode->i_ino); err = fscrypt_encrypt_symlink(inode, symname, len, &disk_link); if (err) goto err_out; err = page_symlink(inode, disk_link.name, disk_link.len); err_out: d_instantiate_new(dentry, inode); /* * Let's flush symlink data in order to avoid broken symlink as much as * possible. Nevertheless, fsyncing is the best way, but there is no * way to get a file descriptor in order to flush that. * * Note that, it needs to do dir->fsync to make this recoverable. * If the symlink path is stored into inline_data, there is no * performance regression. */ if (!err) { filemap_write_and_wait_range(inode->i_mapping, 0, disk_link.len - 1); if (IS_DIRSYNC(dir)) f2fs_sync_fs(sbi->sb, 1); } else { f2fs_unlink(dir, dentry); } f2fs_balance_fs(sbi, true); goto out_free_encrypted_link; out_f2fs_handle_failed_inode: f2fs_handle_failed_inode(inode); out_free_encrypted_link: if (disk_link.name != (unsigned char *)symname) kfree(disk_link.name); return err; } static int f2fs_mkdir(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct inode *inode; int err; if (unlikely(f2fs_cp_error(sbi))) return -EIO; err = f2fs_dquot_initialize(dir); if (err) return err; inode = f2fs_new_inode(idmap, dir, S_IFDIR | mode, NULL); if (IS_ERR(inode)) return PTR_ERR(inode); inode->i_op = &f2fs_dir_inode_operations; inode->i_fop = &f2fs_dir_operations; inode->i_mapping->a_ops = &f2fs_dblock_aops; mapping_set_gfp_mask(inode->i_mapping, GFP_NOFS); set_inode_flag(inode, FI_INC_LINK); f2fs_lock_op(sbi); err = f2fs_add_link(dentry, inode); if (err) goto out_fail; f2fs_unlock_op(sbi); f2fs_alloc_nid_done(sbi, inode->i_ino); d_instantiate_new(dentry, inode); if (IS_DIRSYNC(dir)) f2fs_sync_fs(sbi->sb, 1); f2fs_balance_fs(sbi, true); return 0; out_fail: clear_inode_flag(inode, FI_INC_LINK); f2fs_handle_failed_inode(inode); return err; } static int f2fs_rmdir(struct inode *dir, struct dentry *dentry) { struct inode *inode = d_inode(dentry); if (f2fs_empty_dir(inode)) return f2fs_unlink(dir, dentry); return -ENOTEMPTY; } static int f2fs_mknod(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode, dev_t rdev) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct inode *inode; int err = 0; if (unlikely(f2fs_cp_error(sbi))) return -EIO; if (!f2fs_is_checkpoint_ready(sbi)) return -ENOSPC; err = f2fs_dquot_initialize(dir); if (err) return err; inode = f2fs_new_inode(idmap, dir, mode, NULL); if (IS_ERR(inode)) return PTR_ERR(inode); init_special_inode(inode, inode->i_mode, rdev); inode->i_op = &f2fs_special_inode_operations; f2fs_lock_op(sbi); err = f2fs_add_link(dentry, inode); if (err) goto out; f2fs_unlock_op(sbi); f2fs_alloc_nid_done(sbi, inode->i_ino); d_instantiate_new(dentry, inode); if (IS_DIRSYNC(dir)) f2fs_sync_fs(sbi->sb, 1); f2fs_balance_fs(sbi, true); return 0; out: f2fs_handle_failed_inode(inode); return err; } static int __f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, struct file *file, umode_t mode, bool is_whiteout, struct inode **new_inode, struct f2fs_filename *fname) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); struct inode *inode; int err; err = f2fs_dquot_initialize(dir); if (err) return err; inode = f2fs_new_inode(idmap, dir, mode, NULL); if (IS_ERR(inode)) return PTR_ERR(inode); if (is_whiteout) { init_special_inode(inode, inode->i_mode, WHITEOUT_DEV); inode->i_op = &f2fs_special_inode_operations; } else { inode->i_op = &f2fs_file_inode_operations; inode->i_fop = &f2fs_file_operations; inode->i_mapping->a_ops = &f2fs_dblock_aops; } f2fs_lock_op(sbi); err = f2fs_acquire_orphan_inode(sbi); if (err) goto out; err = f2fs_do_tmpfile(inode, dir, fname); if (err) goto release_out; /* * add this non-linked tmpfile to orphan list, in this way we could * remove all unused data of tmpfile after abnormal power-off. */ f2fs_add_orphan_inode(inode); f2fs_alloc_nid_done(sbi, inode->i_ino); if (is_whiteout) { f2fs_i_links_write(inode, false); spin_lock(&inode->i_lock); inode->i_state |= I_LINKABLE; spin_unlock(&inode->i_lock); } else { if (file) d_tmpfile(file, inode); else f2fs_i_links_write(inode, false); } /* link_count was changed by d_tmpfile as well. */ f2fs_unlock_op(sbi); unlock_new_inode(inode); if (new_inode) *new_inode = inode; f2fs_balance_fs(sbi, true); return 0; release_out: f2fs_release_orphan_inode(sbi); out: f2fs_handle_failed_inode(inode); return err; } static int f2fs_tmpfile(struct mnt_idmap *idmap, struct inode *dir, struct file *file, umode_t mode) { struct f2fs_sb_info *sbi = F2FS_I_SB(dir); int err; if (unlikely(f2fs_cp_error(sbi))) return -EIO; if (!f2fs_is_checkpoint_ready(sbi)) return -ENOSPC; err = __f2fs_tmpfile(idmap, dir, file, mode, false, NULL, NULL); return finish_open_simple(file, err); } static int f2fs_create_whiteout(struct mnt_idmap *idmap, struct inode *dir, struct inode **whiteout, struct f2fs_filename *fname) { return __f2fs_tmpfile(idmap, dir, NULL, S_IFCHR | WHITEOUT_MODE, true, whiteout, fname); } int f2fs_get_tmpfile(struct mnt_idmap *idmap, struct inode *dir, struct inode **new_inode) { return __f2fs_tmpfile(idmap, dir, NULL, S_IFREG, false, new_inode, NULL); } static int f2fs_rename(struct mnt_idmap *idmap, struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { struct f2fs_sb_info *sbi = F2FS_I_SB(old_dir); struct inode *old_inode = d_inode(old_dentry); struct inode *new_inode = d_inode(new_dentry); struct inode *whiteout = NULL; struct page *old_dir_page = NULL; struct page *old_page, *new_page = NULL; struct f2fs_dir_entry *old_dir_entry = NULL; struct f2fs_dir_entry *old_entry; struct f2fs_dir_entry *new_entry; bool old_is_dir = S_ISDIR(old_inode->i_mode); int err; if (unlikely(f2fs_cp_error(sbi))) return -EIO; if (!f2fs_is_checkpoint_ready(sbi)) return -ENOSPC; if (is_inode_flag_set(new_dir, FI_PROJ_INHERIT) && (!projid_eq(F2FS_I(new_dir)->i_projid, F2FS_I(old_dentry->d_inode)->i_projid))) return -EXDEV; /* * If new_inode is null, the below renaming flow will * add a link in old_dir which can convert inline_dir. * After then, if we failed to get the entry due to other * reasons like ENOMEM, we had to remove the new entry. * Instead of adding such the error handling routine, let's * simply convert first here. */ if (old_dir == new_dir && !new_inode) { err = f2fs_try_convert_inline_dir(old_dir, new_dentry); if (err) return err; } if (flags & RENAME_WHITEOUT) { struct f2fs_filename fname; err = f2fs_setup_filename(old_dir, &old_dentry->d_name, 0, &fname); if (err) return err; err = f2fs_create_whiteout(idmap, old_dir, &whiteout, &fname); if (err) return err; } err = f2fs_dquot_initialize(old_dir); if (err) goto out; err = f2fs_dquot_initialize(new_dir); if (err) goto out; if (new_inode) { err = f2fs_dquot_initialize(new_inode); if (err) goto out; } err = -ENOENT; old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_page); if (!old_entry) { if (IS_ERR(old_page)) err = PTR_ERR(old_page); goto out; } if (old_is_dir && old_dir != new_dir) { old_dir_entry = f2fs_parent_dir(old_inode, &old_dir_page); if (!old_dir_entry) { if (IS_ERR(old_dir_page)) err = PTR_ERR(old_dir_page); goto out_old; } } if (new_inode) { err = -ENOTEMPTY; if (old_is_dir && !f2fs_empty_dir(new_inode)) goto out_dir; err = -ENOENT; new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name, &new_page); if (!new_entry) { if (IS_ERR(new_page)) err = PTR_ERR(new_page); goto out_dir; } f2fs_balance_fs(sbi, true); f2fs_lock_op(sbi); err = f2fs_acquire_orphan_inode(sbi); if (err) goto put_out_dir; f2fs_set_link(new_dir, new_entry, new_page, old_inode); new_page = NULL; inode_set_ctime_current(new_inode); f2fs_down_write(&F2FS_I(new_inode)->i_sem); if (old_is_dir) f2fs_i_links_write(new_inode, false); f2fs_i_links_write(new_inode, false); f2fs_up_write(&F2FS_I(new_inode)->i_sem); if (!new_inode->i_nlink) f2fs_add_orphan_inode(new_inode); else f2fs_release_orphan_inode(sbi); } else { f2fs_balance_fs(sbi, true); f2fs_lock_op(sbi); err = f2fs_add_link(new_dentry, old_inode); if (err) { f2fs_unlock_op(sbi); goto out_dir; } if (old_is_dir) f2fs_i_links_write(new_dir, true); } f2fs_down_write(&F2FS_I(old_inode)->i_sem); if (!old_is_dir || whiteout) file_lost_pino(old_inode); else /* adjust dir's i_pino to pass fsck check */ f2fs_i_pino_write(old_inode, new_dir->i_ino); f2fs_up_write(&F2FS_I(old_inode)->i_sem); inode_set_ctime_current(old_inode); f2fs_mark_inode_dirty_sync(old_inode, false); f2fs_delete_entry(old_entry, old_page, old_dir, NULL); old_page = NULL; if (whiteout) { set_inode_flag(whiteout, FI_INC_LINK); err = f2fs_add_link(old_dentry, whiteout); if (err) goto put_out_dir; spin_lock(&whiteout->i_lock); whiteout->i_state &= ~I_LINKABLE; spin_unlock(&whiteout->i_lock); iput(whiteout); } if (old_dir_entry) f2fs_set_link(old_inode, old_dir_entry, old_dir_page, new_dir); if (old_is_dir) f2fs_i_links_write(old_dir, false); if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_STRICT) { f2fs_add_ino_entry(sbi, new_dir->i_ino, TRANS_DIR_INO); if (S_ISDIR(old_inode->i_mode)) f2fs_add_ino_entry(sbi, old_inode->i_ino, TRANS_DIR_INO); } f2fs_unlock_op(sbi); if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir)) f2fs_sync_fs(sbi->sb, 1); f2fs_update_time(sbi, REQ_TIME); return 0; put_out_dir: f2fs_unlock_op(sbi); f2fs_put_page(new_page, 0); out_dir: if (old_dir_entry) f2fs_put_page(old_dir_page, 0); out_old: f2fs_put_page(old_page, 0); out: iput(whiteout); return err; } static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) { struct f2fs_sb_info *sbi = F2FS_I_SB(old_dir); struct inode *old_inode = d_inode(old_dentry); struct inode *new_inode = d_inode(new_dentry); struct page *old_dir_page, *new_dir_page; struct page *old_page, *new_page; struct f2fs_dir_entry *old_dir_entry = NULL, *new_dir_entry = NULL; struct f2fs_dir_entry *old_entry, *new_entry; int old_nlink = 0, new_nlink = 0; int err; if (unlikely(f2fs_cp_error(sbi))) return -EIO; if (!f2fs_is_checkpoint_ready(sbi)) return -ENOSPC; if ((is_inode_flag_set(new_dir, FI_PROJ_INHERIT) && !projid_eq(F2FS_I(new_dir)->i_projid, F2FS_I(old_dentry->d_inode)->i_projid)) || (is_inode_flag_set(new_dir, FI_PROJ_INHERIT) && !projid_eq(F2FS_I(old_dir)->i_projid, F2FS_I(new_dentry->d_inode)->i_projid))) return -EXDEV; err = f2fs_dquot_initialize(old_dir); if (err) goto out; err = f2fs_dquot_initialize(new_dir); if (err) goto out; err = -ENOENT; old_entry = f2fs_find_entry(old_dir, &old_dentry->d_name, &old_page); if (!old_entry) { if (IS_ERR(old_page)) err = PTR_ERR(old_page); goto out; } new_entry = f2fs_find_entry(new_dir, &new_dentry->d_name, &new_page); if (!new_entry) { if (IS_ERR(new_page)) err = PTR_ERR(new_page); goto out_old; } /* prepare for updating ".." directory entry info later */ if (old_dir != new_dir) { if (S_ISDIR(old_inode->i_mode)) { old_dir_entry = f2fs_parent_dir(old_inode, &old_dir_page); if (!old_dir_entry) { if (IS_ERR(old_dir_page)) err = PTR_ERR(old_dir_page); goto out_new; } } if (S_ISDIR(new_inode->i_mode)) { new_dir_entry = f2fs_parent_dir(new_inode, &new_dir_page); if (!new_dir_entry) { if (IS_ERR(new_dir_page)) err = PTR_ERR(new_dir_page); goto out_old_dir; } } } /* * If cross rename between file and directory those are not * in the same directory, we will inc nlink of file's parent * later, so we should check upper boundary of its nlink. */ if ((!old_dir_entry || !new_dir_entry) && old_dir_entry != new_dir_entry) { old_nlink = old_dir_entry ? -1 : 1; new_nlink = -old_nlink; err = -EMLINK; if ((old_nlink > 0 && old_dir->i_nlink >= F2FS_LINK_MAX) || (new_nlink > 0 && new_dir->i_nlink >= F2FS_LINK_MAX)) goto out_new_dir; } f2fs_balance_fs(sbi, true); f2fs_lock_op(sbi); /* update ".." directory entry info of old dentry */ if (old_dir_entry) f2fs_set_link(old_inode, old_dir_entry, old_dir_page, new_dir); /* update ".." directory entry info of new dentry */ if (new_dir_entry) f2fs_set_link(new_inode, new_dir_entry, new_dir_page, old_dir); /* update directory entry info of old dir inode */ f2fs_set_link(old_dir, old_entry, old_page, new_inode); f2fs_down_write(&F2FS_I(old_inode)->i_sem); if (!old_dir_entry) file_lost_pino(old_inode); else /* adjust dir's i_pino to pass fsck check */ f2fs_i_pino_write(old_inode, new_dir->i_ino); f2fs_up_write(&F2FS_I(old_inode)->i_sem); inode_set_ctime_current(old_dir); if (old_nlink) { f2fs_down_write(&F2FS_I(old_dir)->i_sem); f2fs_i_links_write(old_dir, old_nlink > 0); f2fs_up_write(&F2FS_I(old_dir)->i_sem); } f2fs_mark_inode_dirty_sync(old_dir, false); /* update directory entry info of new dir inode */ f2fs_set_link(new_dir, new_entry, new_page, old_inode); f2fs_down_write(&F2FS_I(new_inode)->i_sem); if (!new_dir_entry) file_lost_pino(new_inode); else /* adjust dir's i_pino to pass fsck check */ f2fs_i_pino_write(new_inode, old_dir->i_ino); f2fs_up_write(&F2FS_I(new_inode)->i_sem); inode_set_ctime_current(new_dir); if (new_nlink) { f2fs_down_write(&F2FS_I(new_dir)->i_sem); f2fs_i_links_write(new_dir, new_nlink > 0); f2fs_up_write(&F2FS_I(new_dir)->i_sem); } f2fs_mark_inode_dirty_sync(new_dir, false); if (F2FS_OPTION(sbi).fsync_mode == FSYNC_MODE_STRICT) { f2fs_add_ino_entry(sbi, old_dir->i_ino, TRANS_DIR_INO); f2fs_add_ino_entry(sbi, new_dir->i_ino, TRANS_DIR_INO); } f2fs_unlock_op(sbi); if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir)) f2fs_sync_fs(sbi->sb, 1); f2fs_update_time(sbi, REQ_TIME); return 0; out_new_dir: if (new_dir_entry) { f2fs_put_page(new_dir_page, 0); } out_old_dir: if (old_dir_entry) { f2fs_put_page(old_dir_page, 0); } out_new: f2fs_put_page(new_page, 0); out_old: f2fs_put_page(old_page, 0); out: return err; } static int f2fs_rename2(struct mnt_idmap *idmap, struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) { int err; if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) return -EINVAL; trace_f2fs_rename_start(old_dir, old_dentry, new_dir, new_dentry, flags); err = fscrypt_prepare_rename(old_dir, old_dentry, new_dir, new_dentry, flags); if (err) return err; if (flags & RENAME_EXCHANGE) err = f2fs_cross_rename(old_dir, old_dentry, new_dir, new_dentry); else /* * VFS has already handled the new dentry existence case, * here, we just deal with "RENAME_NOREPLACE" as regular rename. */ err = f2fs_rename(idmap, old_dir, old_dentry, new_dir, new_dentry, flags); trace_f2fs_rename_end(old_dentry, new_dentry, flags, err); return err; } static const char *f2fs_encrypted_get_link(struct dentry *dentry, struct inode *inode, struct delayed_call *done) { struct page *page; const char *target; if (!dentry) return ERR_PTR(-ECHILD); page = read_mapping_page(inode->i_mapping, 0, NULL); if (IS_ERR(page)) return ERR_CAST(page); target = fscrypt_get_symlink(inode, page_address(page), inode->i_sb->s_blocksize, done); put_page(page); return target; } static int f2fs_encrypted_symlink_getattr(struct mnt_idmap *idmap, const struct path *path, struct kstat *stat, u32 request_mask, unsigned int query_flags) { f2fs_getattr(idmap, path, stat, request_mask, query_flags); return fscrypt_symlink_getattr(path, stat); } const struct inode_operations f2fs_encrypted_symlink_inode_operations = { .get_link = f2fs_encrypted_get_link, .getattr = f2fs_encrypted_symlink_getattr, .setattr = f2fs_setattr, .listxattr = f2fs_listxattr, }; const struct inode_operations f2fs_dir_inode_operations = { .create = f2fs_create, .lookup = f2fs_lookup, .link = f2fs_link, .unlink = f2fs_unlink, .symlink = f2fs_symlink, .mkdir = f2fs_mkdir, .rmdir = f2fs_rmdir, .mknod = f2fs_mknod, .rename = f2fs_rename2, .tmpfile = f2fs_tmpfile, .getattr = f2fs_getattr, .setattr = f2fs_setattr, .get_inode_acl = f2fs_get_acl, .set_acl = f2fs_set_acl, .listxattr = f2fs_listxattr, .fiemap = f2fs_fiemap, .fileattr_get = f2fs_fileattr_get, .fileattr_set = f2fs_fileattr_set, }; const struct inode_operations f2fs_symlink_inode_operations = { .get_link = f2fs_get_link, .getattr = f2fs_getattr, .setattr = f2fs_setattr, .listxattr = f2fs_listxattr, }; const struct inode_operations f2fs_special_inode_operations = { .getattr = f2fs_getattr, .setattr = f2fs_setattr, .get_inode_acl = f2fs_get_acl, .set_acl = f2fs_set_acl, .listxattr = f2fs_listxattr, }; |
| 1 1 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Driver for the Auvitek USB bridge * * Copyright (c) 2008 Steven Toth <stoth@linuxtv.org> */ #include "au0828.h" #include "au8522.h" #include <linux/module.h> #include <linux/slab.h> #include <linux/videodev2.h> #include <media/v4l2-common.h> #include <linux/mutex.h> /* Due to enum tuner_pad_index */ #include <media/tuner.h> /* * 1 = General debug messages * 2 = USB handling * 4 = I2C related * 8 = Bridge related * 16 = IR related */ int au0828_debug; module_param_named(debug, au0828_debug, int, 0644); MODULE_PARM_DESC(debug, "set debug bitmask: 1=general, 2=USB, 4=I2C, 8=bridge, 16=IR"); static unsigned int disable_usb_speed_check; module_param(disable_usb_speed_check, int, 0444); MODULE_PARM_DESC(disable_usb_speed_check, "override min bandwidth requirement of 480M bps"); #define _AU0828_BULKPIPE 0x03 #define _BULKPIPESIZE 0xffff static int send_control_msg(struct au0828_dev *dev, u16 request, u32 value, u16 index); static int recv_control_msg(struct au0828_dev *dev, u16 request, u32 value, u16 index, unsigned char *cp, u16 size); /* USB Direction */ #define CMD_REQUEST_IN 0x00 #define CMD_REQUEST_OUT 0x01 u32 au0828_readreg(struct au0828_dev *dev, u16 reg) { u8 result = 0; recv_control_msg(dev, CMD_REQUEST_IN, 0, reg, &result, 1); dprintk(8, "%s(0x%04x) = 0x%02x\n", __func__, reg, result); return result; } u32 au0828_writereg(struct au0828_dev *dev, u16 reg, u32 val) { dprintk(8, "%s(0x%04x, 0x%02x)\n", __func__, reg, val); return send_control_msg(dev, CMD_REQUEST_OUT, val, reg); } static int send_control_msg(struct au0828_dev *dev, u16 request, u32 value, u16 index) { int status = -ENODEV; if (dev->usbdev) { /* cp must be memory that has been allocated by kmalloc */ status = usb_control_msg(dev->usbdev, usb_sndctrlpipe(dev->usbdev, 0), request, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE, value, index, NULL, 0, 1000); status = min(status, 0); if (status < 0) { pr_err("%s() Failed sending control message, error %d.\n", __func__, status); } } return status; } static int recv_control_msg(struct au0828_dev *dev, u16 request, u32 value, u16 index, unsigned char *cp, u16 size) { int status = -ENODEV; mutex_lock(&dev->mutex); if (dev->usbdev) { status = usb_control_msg(dev->usbdev, usb_rcvctrlpipe(dev->usbdev, 0), request, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, value, index, dev->ctrlmsg, size, 1000); status = min(status, 0); if (status < 0) { pr_err("%s() Failed receiving control message, error %d.\n", __func__, status); } /* the host controller requires heap allocated memory, which is why we didn't just pass "cp" into usb_control_msg */ memcpy(cp, dev->ctrlmsg, size); } mutex_unlock(&dev->mutex); return status; } #ifdef CONFIG_MEDIA_CONTROLLER static void au0828_media_graph_notify(struct media_entity *new, void *notify_data); #endif static void au0828_unregister_media_device(struct au0828_dev *dev) { #ifdef CONFIG_MEDIA_CONTROLLER struct media_device *mdev = dev->media_dev; struct media_entity_notify *notify, *nextp; if (!mdev || !media_devnode_is_registered(mdev->devnode)) return; /* Remove au0828 entity_notify callbacks */ list_for_each_entry_safe(notify, nextp, &mdev->entity_notify, list) { if (notify->notify != au0828_media_graph_notify) continue; media_device_unregister_entity_notify(mdev, notify); } /* clear enable_source, disable_source */ mutex_lock(&mdev->graph_mutex); dev->media_dev->source_priv = NULL; dev->media_dev->enable_source = NULL; dev->media_dev->disable_source = NULL; mutex_unlock(&mdev->graph_mutex); media_device_delete(dev->media_dev, KBUILD_MODNAME, THIS_MODULE); dev->media_dev = NULL; #endif } void au0828_usb_release(struct au0828_dev *dev) { au0828_unregister_media_device(dev); /* I2C */ au0828_i2c_unregister(dev); kfree(dev); } static void au0828_usb_disconnect(struct usb_interface *interface) { struct au0828_dev *dev = usb_get_intfdata(interface); dprintk(1, "%s()\n", __func__); /* there is a small window after disconnect, before dev->usbdev is NULL, for poll (e.g: IR) try to access the device and fill the dmesg with error messages. Set the status so poll routines can check and avoid access after disconnect. */ set_bit(DEV_DISCONNECTED, &dev->dev_state); au0828_rc_unregister(dev); /* Digital TV */ au0828_dvb_unregister(dev); usb_set_intfdata(interface, NULL); mutex_lock(&dev->mutex); dev->usbdev = NULL; mutex_unlock(&dev->mutex); if (au0828_analog_unregister(dev)) { /* * No need to call au0828_usb_release() if V4L2 is enabled, * as this is already called via au0828_usb_v4l2_release() */ return; } au0828_usb_release(dev); } static int au0828_media_device_init(struct au0828_dev *dev, struct usb_device *udev) { #ifdef CONFIG_MEDIA_CONTROLLER struct media_device *mdev; mdev = media_device_usb_allocate(udev, KBUILD_MODNAME, THIS_MODULE); if (IS_ERR(mdev)) return PTR_ERR(mdev); dev->media_dev = mdev; #endif return 0; } #ifdef CONFIG_MEDIA_CONTROLLER static void au0828_media_graph_notify(struct media_entity *new, void *notify_data) { struct au0828_dev *dev = notify_data; int ret; struct media_entity *entity, *mixer = NULL, *decoder = NULL; if (!new) { /* * Called during au0828 probe time to connect * entities that were created prior to registering * the notify handler. Find mixer and decoder. */ media_device_for_each_entity(entity, dev->media_dev) { if (entity->function == MEDIA_ENT_F_AUDIO_MIXER) mixer = entity; else if (entity->function == MEDIA_ENT_F_ATV_DECODER) decoder = entity; } goto create_link; } switch (new->function) { case MEDIA_ENT_F_AUDIO_MIXER: mixer = new; if (dev->decoder) decoder = dev->decoder; break; case MEDIA_ENT_F_ATV_DECODER: /* In case, Mixer is added first, find mixer and create link */ media_device_for_each_entity(entity, dev->media_dev) { if (entity->function == MEDIA_ENT_F_AUDIO_MIXER) mixer = entity; } decoder = new; break; default: break; } create_link: if (decoder && mixer) { ret = media_get_pad_index(decoder, MEDIA_PAD_FL_SOURCE, PAD_SIGNAL_AUDIO); if (ret >= 0) ret = media_create_pad_link(decoder, ret, mixer, 0, MEDIA_LNK_FL_ENABLED); if (ret < 0) dev_err(&dev->usbdev->dev, "Mixer Pad Link Create Error: %d\n", ret); } } static bool au0828_is_link_shareable(struct media_entity *owner, struct media_entity *entity) { bool shareable = false; /* Tuner link can be shared by audio, video, and VBI */ switch (owner->function) { case MEDIA_ENT_F_IO_V4L: case MEDIA_ENT_F_AUDIO_CAPTURE: case MEDIA_ENT_F_IO_VBI: if (entity->function == MEDIA_ENT_F_IO_V4L || entity->function == MEDIA_ENT_F_AUDIO_CAPTURE || entity->function == MEDIA_ENT_F_IO_VBI) shareable = true; break; case MEDIA_ENT_F_DTV_DEMOD: default: break; } return shareable; } /* Callers should hold graph_mutex */ static int au0828_enable_source(struct media_entity *entity, struct media_pipeline *pipe) { struct media_entity *source, *find_source; struct media_entity *sink; struct media_link *link, *found_link = NULL; int ret = 0; struct media_device *mdev = entity->graph_obj.mdev; struct au0828_dev *dev; if (!mdev) return -ENODEV; dev = mdev->source_priv; /* * For Audio and V4L2 entity, find the link to which decoder * is the sink. Look for an active link between decoder and * source (tuner/s-video/Composite), if one exists, nothing * to do. If not, look for any active links between source * and any other entity. If one exists, source is busy. If * source is free, setup link and start pipeline from source. * For DVB FE entity, the source for the link is the tuner. * Check if tuner is available and setup link and start * pipeline. */ if (entity->function == MEDIA_ENT_F_DTV_DEMOD) { sink = entity; find_source = dev->tuner; } else { /* Analog isn't configured or register failed */ if (!dev->decoder) { ret = -ENODEV; goto end; } sink = dev->decoder; /* * Default input is tuner and default input_type * is AU0828_VMUX_TELEVISION. * * There is a problem when s_input is called to * change the default input. s_input will try to * enable_source before attempting to change the * input on the device, and will end up enabling * default source which is tuner. * * Additional logic is necessary in au0828 to detect * that the input has changed and enable the right * source. au0828 handles this case in its s_input. * It will disable the old source and enable the new * source. * */ if (dev->input_type == AU0828_VMUX_TELEVISION) find_source = dev->tuner; else if (dev->input_type == AU0828_VMUX_SVIDEO || dev->input_type == AU0828_VMUX_COMPOSITE) find_source = &dev->input_ent[dev->input_type]; else { /* unknown input - let user select input */ ret = 0; goto end; } } /* Is there an active link between sink and source */ if (dev->active_link) { if (dev->active_link_owner == entity) { /* This check is necessary to handle multiple * enable_source calls from v4l_ioctls during * the course of video/vbi application run-time. */ pr_debug("%s already owns the tuner\n", entity->name); ret = 0; goto end; } else if (au0828_is_link_shareable(dev->active_link_owner, entity)) { /* Either ALSA or Video own tuner. Sink is the same * for both. Allow sharing the active link between * their common source (tuner) and sink (decoder). * Starting pipeline between sharing entity and sink * will fail with pipe mismatch, while owner has an * active pipeline. Switch pipeline ownership from * user to owner when owner disables the source. */ dev->active_link_shared = true; /* save the user info to use from disable */ dev->active_link_user = entity; dev->active_link_user_pipe = pipe; pr_debug("%s owns the tuner %s can share!\n", dev->active_link_owner->name, entity->name); ret = 0; goto end; } else { ret = -EBUSY; goto end; } } list_for_each_entry(link, &sink->links, list) { /* Check sink, and source */ if (link->sink->entity == sink && link->source->entity == find_source) { found_link = link; break; } } if (!found_link) { ret = -ENODEV; goto end; } /* activate link between source and sink and start pipeline */ source = found_link->source->entity; ret = __media_entity_setup_link(found_link, MEDIA_LNK_FL_ENABLED); if (ret) { pr_err("Activate link from %s->%s. Error %d\n", source->name, sink->name, ret); goto end; } ret = __media_pipeline_start(entity->pads, pipe); if (ret) { pr_err("Start Pipeline: %s->%s Error %d\n", source->name, entity->name, ret); ret = __media_entity_setup_link(found_link, 0); if (ret) pr_err("Deactivate link Error %d\n", ret); goto end; } /* save link state to allow audio and video share the link * and not disable the link while the other is using it. * active_link_owner is used to deactivate the link. */ dev->active_link = found_link; dev->active_link_owner = entity; dev->active_source = source; dev->active_sink = sink; pr_info("Enabled Source: %s->%s->%s Ret %d\n", dev->active_source->name, dev->active_sink->name, dev->active_link_owner->name, ret); end: pr_debug("%s end: ent:%s fnc:%d ret %d\n", __func__, entity->name, entity->function, ret); return ret; } /* Callers should hold graph_mutex */ static void au0828_disable_source(struct media_entity *entity) { int ret = 0; struct media_device *mdev = entity->graph_obj.mdev; struct au0828_dev *dev; if (!mdev) return; dev = mdev->source_priv; if (!dev->active_link) return; /* link is active - stop pipeline from source * (tuner/s-video/Composite) to the entity * When DVB/s-video/Composite owns tuner, it won't be in * shared state. */ if (dev->active_link->sink->entity == dev->active_sink && dev->active_link->source->entity == dev->active_source) { /* * Prevent video from deactivating link when audio * has active pipeline and vice versa. In addition * handle the case when more than one video/vbi * application is sharing the link. */ bool owner_is_audio = false; if (dev->active_link_owner->function == MEDIA_ENT_F_AUDIO_CAPTURE) owner_is_audio = true; if (dev->active_link_shared) { pr_debug("Shared link owner %s user %s %d\n", dev->active_link_owner->name, entity->name, dev->users); /* Handle video device users > 1 * When audio owns the shared link with * more than one video users, avoid * disabling the source and/or switching * the owner until the last disable_source * call from video _close(). Use dev->users to * determine when to switch/disable. */ if (dev->active_link_owner != entity) { /* video device has users > 1 */ if (owner_is_audio && dev->users > 1) return; dev->active_link_user = NULL; dev->active_link_user_pipe = NULL; dev->active_link_shared = false; return; } /* video owns the link and has users > 1 */ if (!owner_is_audio && dev->users > 1) return; /* stop pipeline */ __media_pipeline_stop(dev->active_link_owner->pads); pr_debug("Pipeline stop for %s\n", dev->active_link_owner->name); ret = __media_pipeline_start( dev->active_link_user->pads, dev->active_link_user_pipe); if (ret) { pr_err("Start Pipeline: %s->%s %d\n", dev->active_source->name, dev->active_link_user->name, ret); goto deactivate_link; } /* link user is now the owner */ dev->active_link_owner = dev->active_link_user; dev->active_link_user = NULL; dev->active_link_user_pipe = NULL; dev->active_link_shared = false; pr_debug("Pipeline started for %s\n", dev->active_link_owner->name); return; } else if (!owner_is_audio && dev->users > 1) /* video/vbi owns the link and has users > 1 */ return; if (dev->active_link_owner != entity) return; /* stop pipeline */ __media_pipeline_stop(dev->active_link_owner->pads); pr_debug("Pipeline stop for %s\n", dev->active_link_owner->name); deactivate_link: ret = __media_entity_setup_link(dev->active_link, 0); if (ret) pr_err("Deactivate link Error %d\n", ret); pr_info("Disabled Source: %s->%s->%s Ret %d\n", dev->active_source->name, dev->active_sink->name, dev->active_link_owner->name, ret); dev->active_link = NULL; dev->active_link_owner = NULL; dev->active_source = NULL; dev->active_sink = NULL; dev->active_link_shared = false; dev->active_link_user = NULL; } } #endif static int au0828_media_device_register(struct au0828_dev *dev, struct usb_device *udev) { #ifdef CONFIG_MEDIA_CONTROLLER int ret; struct media_entity *entity, *demod = NULL; struct media_link *link; if (!dev->media_dev) return 0; if (!media_devnode_is_registered(dev->media_dev->devnode)) { /* register media device */ ret = media_device_register(dev->media_dev); if (ret) { media_device_delete(dev->media_dev, KBUILD_MODNAME, THIS_MODULE); dev->media_dev = NULL; dev_err(&udev->dev, "Media Device Register Error: %d\n", ret); return ret; } } else { /* * Call au0828_media_graph_notify() to connect * audio graph to our graph. In this case, audio * driver registered the device and there is no * entity_notify to be called when new entities * are added. Invoke it now. */ au0828_media_graph_notify(NULL, (void *) dev); } /* * Find tuner, decoder and demod. * * The tuner and decoder should be cached, as they'll be used by * au0828_enable_source. * * It also needs to disable the link between tuner and * decoder/demod, to avoid disable step when tuner is requested * by video or audio. Note that this step can't be done until dvb * graph is created during dvb register. */ media_device_for_each_entity(entity, dev->media_dev) { switch (entity->function) { case MEDIA_ENT_F_TUNER: dev->tuner = entity; break; case MEDIA_ENT_F_ATV_DECODER: dev->decoder = entity; break; case MEDIA_ENT_F_DTV_DEMOD: demod = entity; break; } } /* Disable link between tuner->demod and/or tuner->decoder */ if (dev->tuner) { list_for_each_entry(link, &dev->tuner->links, list) { if (demod && link->sink->entity == demod) media_entity_setup_link(link, 0); if (dev->decoder && link->sink->entity == dev->decoder) media_entity_setup_link(link, 0); } } /* register entity_notify callback */ dev->entity_notify.notify_data = (void *) dev; dev->entity_notify.notify = (void *) au0828_media_graph_notify; media_device_register_entity_notify(dev->media_dev, &dev->entity_notify); /* set enable_source */ mutex_lock(&dev->media_dev->graph_mutex); dev->media_dev->source_priv = (void *) dev; dev->media_dev->enable_source = au0828_enable_source; dev->media_dev->disable_source = au0828_disable_source; mutex_unlock(&dev->media_dev->graph_mutex); #endif return 0; } static int au0828_usb_probe(struct usb_interface *interface, const struct usb_device_id *id) { int ifnum; int retval = 0; struct au0828_dev *dev; struct usb_device *usbdev = interface_to_usbdev(interface); ifnum = interface->altsetting->desc.bInterfaceNumber; if (ifnum != 0) return -ENODEV; dprintk(1, "%s() vendor id 0x%x device id 0x%x ifnum:%d\n", __func__, le16_to_cpu(usbdev->descriptor.idVendor), le16_to_cpu(usbdev->descriptor.idProduct), ifnum); /* * Make sure we have 480 Mbps of bandwidth, otherwise things like * video stream wouldn't likely work, since 12 Mbps is generally * not enough even for most Digital TV streams. */ if (usbdev->speed != USB_SPEED_HIGH && disable_usb_speed_check == 0) { pr_err("au0828: Device initialization failed.\n"); pr_err("au0828: Device must be connected to a high-speed USB 2.0 port.\n"); return -ENODEV; } dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (dev == NULL) { pr_err("%s() Unable to allocate memory\n", __func__); return -ENOMEM; } mutex_init(&dev->lock); mutex_lock(&dev->lock); mutex_init(&dev->mutex); mutex_init(&dev->dvb.lock); dev->usbdev = usbdev; dev->boardnr = id->driver_info; dev->board = au0828_boards[dev->boardnr]; /* Initialize the media controller */ retval = au0828_media_device_init(dev, usbdev); if (retval) { pr_err("%s() au0828_media_device_init failed\n", __func__); mutex_unlock(&dev->lock); kfree(dev); return retval; } retval = au0828_v4l2_device_register(interface, dev); if (retval) { au0828_usb_v4l2_media_release(dev); mutex_unlock(&dev->lock); kfree(dev); return retval; } /* Power Up the bridge */ au0828_write(dev, REG_600, 1 << 4); /* Bring up the GPIO's and supporting devices */ au0828_gpio_setup(dev); /* I2C */ au0828_i2c_register(dev); /* Setup */ au0828_card_setup(dev); /* * Store the pointer to the au0828_dev so it can be accessed in * au0828_usb_disconnect */ usb_set_intfdata(interface, dev); /* Analog TV */ retval = au0828_analog_register(dev, interface); if (retval) { pr_err("%s() au0828_analog_register failed to register on V4L2\n", __func__); mutex_unlock(&dev->lock); goto done; } /* Digital TV */ retval = au0828_dvb_register(dev); if (retval) pr_err("%s() au0828_dvb_register failed\n", __func__); /* Remote controller */ au0828_rc_register(dev); pr_info("Registered device AU0828 [%s]\n", dev->board.name == NULL ? "Unset" : dev->board.name); mutex_unlock(&dev->lock); retval = au0828_media_device_register(dev, usbdev); done: if (retval < 0) au0828_usb_disconnect(interface); return retval; } static int au0828_suspend(struct usb_interface *interface, pm_message_t message) { struct au0828_dev *dev = usb_get_intfdata(interface); if (!dev) return 0; pr_info("Suspend\n"); au0828_rc_suspend(dev); au0828_v4l2_suspend(dev); au0828_dvb_suspend(dev); /* FIXME: should suspend also ATV/DTV */ return 0; } static int au0828_resume(struct usb_interface *interface) { struct au0828_dev *dev = usb_get_intfdata(interface); if (!dev) return 0; pr_info("Resume\n"); /* Power Up the bridge */ au0828_write(dev, REG_600, 1 << 4); /* Bring up the GPIO's and supporting devices */ au0828_gpio_setup(dev); au0828_rc_resume(dev); au0828_v4l2_resume(dev); au0828_dvb_resume(dev); /* FIXME: should resume also ATV/DTV */ return 0; } static struct usb_driver au0828_usb_driver = { .name = KBUILD_MODNAME, .probe = au0828_usb_probe, .disconnect = au0828_usb_disconnect, .id_table = au0828_usb_id_table, .suspend = au0828_suspend, .resume = au0828_resume, .reset_resume = au0828_resume, }; static int __init au0828_init(void) { int ret; if (au0828_debug & 1) pr_info("%s() Debugging is enabled\n", __func__); if (au0828_debug & 2) pr_info("%s() USB Debugging is enabled\n", __func__); if (au0828_debug & 4) pr_info("%s() I2C Debugging is enabled\n", __func__); if (au0828_debug & 8) pr_info("%s() Bridge Debugging is enabled\n", __func__); if (au0828_debug & 16) pr_info("%s() IR Debugging is enabled\n", __func__); pr_info("au0828 driver loaded\n"); ret = usb_register(&au0828_usb_driver); if (ret) pr_err("usb_register failed, error = %d\n", ret); return ret; } static void __exit au0828_exit(void) { usb_deregister(&au0828_usb_driver); } module_init(au0828_init); module_exit(au0828_exit); MODULE_DESCRIPTION("Driver for Auvitek AU0828 based products"); MODULE_AUTHOR("Steven Toth <stoth@linuxtv.org>"); MODULE_LICENSE("GPL"); MODULE_VERSION("0.0.3"); |
| 9 9 1 1 1 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 | // SPDX-License-Identifier: GPL-2.0-or-later /* * * Copyright (C) Jonathan Naylor G4KLX (g4klx@g4klx.demon.co.uk) */ #include <linux/errno.h> #include <linux/types.h> #include <linux/socket.h> #include <linux/slab.h> #include <linux/in.h> #include <linux/kernel.h> #include <linux/timer.h> #include <linux/string.h> #include <linux/sockios.h> #include <linux/net.h> #include <linux/spinlock.h> #include <net/ax25.h> #include <linux/inet.h> #include <linux/netdevice.h> #include <linux/if_arp.h> #include <linux/skbuff.h> #include <net/sock.h> #include <linux/uaccess.h> #include <linux/fcntl.h> #include <linux/list.h> #include <linux/mm.h> #include <linux/interrupt.h> #include <linux/init.h> static LIST_HEAD(ax25_dev_list); DEFINE_SPINLOCK(ax25_dev_lock); ax25_dev *ax25_addr_ax25dev(ax25_address *addr) { ax25_dev *ax25_dev, *res = NULL; spin_lock_bh(&ax25_dev_lock); list_for_each_entry(ax25_dev, &ax25_dev_list, list) if (ax25cmp(addr, (const ax25_address *)ax25_dev->dev->dev_addr) == 0) { res = ax25_dev; ax25_dev_hold(ax25_dev); break; } spin_unlock_bh(&ax25_dev_lock); return res; } /* * This is called when an interface is brought up. These are * reasonable defaults. */ void ax25_dev_device_up(struct net_device *dev) { ax25_dev *ax25_dev; ax25_dev = kzalloc(sizeof(*ax25_dev), GFP_KERNEL); if (!ax25_dev) { printk(KERN_ERR "AX.25: ax25_dev_device_up - out of memory\n"); return; } refcount_set(&ax25_dev->refcount, 1); ax25_dev->dev = dev; netdev_hold(dev, &ax25_dev->dev_tracker, GFP_KERNEL); ax25_dev->forward = NULL; ax25_dev->device_up = true; ax25_dev->values[AX25_VALUES_IPDEFMODE] = AX25_DEF_IPDEFMODE; ax25_dev->values[AX25_VALUES_AXDEFMODE] = AX25_DEF_AXDEFMODE; ax25_dev->values[AX25_VALUES_BACKOFF] = AX25_DEF_BACKOFF; ax25_dev->values[AX25_VALUES_CONMODE] = AX25_DEF_CONMODE; ax25_dev->values[AX25_VALUES_WINDOW] = AX25_DEF_WINDOW; ax25_dev->values[AX25_VALUES_EWINDOW] = AX25_DEF_EWINDOW; ax25_dev->values[AX25_VALUES_T1] = AX25_DEF_T1; ax25_dev->values[AX25_VALUES_T2] = AX25_DEF_T2; ax25_dev->values[AX25_VALUES_T3] = AX25_DEF_T3; ax25_dev->values[AX25_VALUES_IDLE] = AX25_DEF_IDLE; ax25_dev->values[AX25_VALUES_N2] = AX25_DEF_N2; ax25_dev->values[AX25_VALUES_PACLEN] = AX25_DEF_PACLEN; ax25_dev->values[AX25_VALUES_PROTOCOL] = AX25_DEF_PROTOCOL; #ifdef CONFIG_AX25_DAMA_SLAVE ax25_dev->values[AX25_VALUES_DS_TIMEOUT]= AX25_DEF_DS_TIMEOUT; #endif #if defined(CONFIG_AX25_DAMA_SLAVE) || defined(CONFIG_AX25_DAMA_MASTER) ax25_ds_setup_timer(ax25_dev); #endif spin_lock_bh(&ax25_dev_lock); list_add(&ax25_dev->list, &ax25_dev_list); dev->ax25_ptr = ax25_dev; spin_unlock_bh(&ax25_dev_lock); ax25_register_dev_sysctl(ax25_dev); } void ax25_dev_device_down(struct net_device *dev) { ax25_dev *s, *ax25_dev; if ((ax25_dev = ax25_dev_ax25dev(dev)) == NULL) return; ax25_unregister_dev_sysctl(ax25_dev); spin_lock_bh(&ax25_dev_lock); #ifdef CONFIG_AX25_DAMA_SLAVE timer_shutdown_sync(&ax25_dev->dama.slave_timer); #endif /* * Remove any packet forwarding that points to this device. */ list_for_each_entry(s, &ax25_dev_list, list) if (s->forward == dev) s->forward = NULL; list_for_each_entry(s, &ax25_dev_list, list) { if (s == ax25_dev) { list_del(&s->list); break; } } dev->ax25_ptr = NULL; spin_unlock_bh(&ax25_dev_lock); netdev_put(dev, &ax25_dev->dev_tracker); ax25_dev_put(ax25_dev); } int ax25_fwd_ioctl(unsigned int cmd, struct ax25_fwd_struct *fwd) { ax25_dev *ax25_dev, *fwd_dev; if ((ax25_dev = ax25_addr_ax25dev(&fwd->port_from)) == NULL) return -EINVAL; switch (cmd) { case SIOCAX25ADDFWD: fwd_dev = ax25_addr_ax25dev(&fwd->port_to); if (!fwd_dev) { ax25_dev_put(ax25_dev); return -EINVAL; } if (ax25_dev->forward) { ax25_dev_put(fwd_dev); ax25_dev_put(ax25_dev); return -EINVAL; } ax25_dev->forward = fwd_dev->dev; ax25_dev_put(fwd_dev); ax25_dev_put(ax25_dev); break; case SIOCAX25DELFWD: if (!ax25_dev->forward) { ax25_dev_put(ax25_dev); return -EINVAL; } ax25_dev->forward = NULL; ax25_dev_put(ax25_dev); break; default: ax25_dev_put(ax25_dev); return -EINVAL; } return 0; } struct net_device *ax25_fwd_dev(struct net_device *dev) { ax25_dev *ax25_dev; if ((ax25_dev = ax25_dev_ax25dev(dev)) == NULL) return dev; if (ax25_dev->forward == NULL) return dev; return ax25_dev->forward; } /* * Free all memory associated with device structures. */ void __exit ax25_dev_free(void) { ax25_dev *s, *n; spin_lock_bh(&ax25_dev_lock); list_for_each_entry_safe(s, n, &ax25_dev_list, list) { netdev_put(s->dev, &s->dev_tracker); list_del(&s->list); ax25_dev_put(s); } spin_unlock_bh(&ax25_dev_lock); } |
| 1 1 1 17 8 9 10 9 2 1 1 1 1 1 1 1 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 | // SPDX-License-Identifier: GPL-2.0 /* * Support for async notification of waitid */ #include <linux/kernel.h> #include <linux/errno.h> #include <linux/fs.h> #include <linux/file.h> #include <linux/compat.h> #include <linux/io_uring.h> #include <uapi/linux/io_uring.h> #include "io_uring.h" #include "cancel.h" #include "waitid.h" #include "../kernel/exit.h" static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts); #define IO_WAITID_CANCEL_FLAG BIT(31) #define IO_WAITID_REF_MASK GENMASK(30, 0) struct io_waitid { struct file *file; int which; pid_t upid; int options; atomic_t refs; struct wait_queue_head *head; struct siginfo __user *infop; struct waitid_info info; }; static void io_waitid_free(struct io_kiocb *req) { struct io_waitid_async *iwa = req->async_data; put_pid(iwa->wo.wo_pid); kfree(req->async_data); req->async_data = NULL; req->flags &= ~REQ_F_ASYNC_DATA; } #ifdef CONFIG_COMPAT static bool io_waitid_compat_copy_si(struct io_waitid *iw, int signo) { struct compat_siginfo __user *infop; bool ret; infop = (struct compat_siginfo __user *) iw->infop; if (!user_write_access_begin(infop, sizeof(*infop))) return false; unsafe_put_user(signo, &infop->si_signo, Efault); unsafe_put_user(0, &infop->si_errno, Efault); unsafe_put_user(iw->info.cause, &infop->si_code, Efault); unsafe_put_user(iw->info.pid, &infop->si_pid, Efault); unsafe_put_user(iw->info.uid, &infop->si_uid, Efault); unsafe_put_user(iw->info.status, &infop->si_status, Efault); ret = true; done: user_write_access_end(); return ret; Efault: ret = false; goto done; } #endif static bool io_waitid_copy_si(struct io_kiocb *req, int signo) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); bool ret; if (!iw->infop) return true; #ifdef CONFIG_COMPAT if (req->ctx->compat) return io_waitid_compat_copy_si(iw, signo); #endif if (!user_write_access_begin(iw->infop, sizeof(*iw->infop))) return false; unsafe_put_user(signo, &iw->infop->si_signo, Efault); unsafe_put_user(0, &iw->infop->si_errno, Efault); unsafe_put_user(iw->info.cause, &iw->infop->si_code, Efault); unsafe_put_user(iw->info.pid, &iw->infop->si_pid, Efault); unsafe_put_user(iw->info.uid, &iw->infop->si_uid, Efault); unsafe_put_user(iw->info.status, &iw->infop->si_status, Efault); ret = true; done: user_write_access_end(); return ret; Efault: ret = false; goto done; } static int io_waitid_finish(struct io_kiocb *req, int ret) { int signo = 0; if (ret > 0) { signo = SIGCHLD; ret = 0; } if (!io_waitid_copy_si(req, signo)) ret = -EFAULT; io_waitid_free(req); return ret; } static void io_waitid_complete(struct io_kiocb *req, int ret) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); struct io_tw_state ts = {}; /* anyone completing better be holding a reference */ WARN_ON_ONCE(!(atomic_read(&iw->refs) & IO_WAITID_REF_MASK)); lockdep_assert_held(&req->ctx->uring_lock); hlist_del_init(&req->hash_node); ret = io_waitid_finish(req, ret); if (ret < 0) req_set_fail(req); io_req_set_res(req, ret, 0); io_req_task_complete(req, &ts); } static bool __io_waitid_cancel(struct io_ring_ctx *ctx, struct io_kiocb *req) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); struct io_waitid_async *iwa = req->async_data; /* * Mark us canceled regardless of ownership. This will prevent a * potential retry from a spurious wakeup. */ atomic_or(IO_WAITID_CANCEL_FLAG, &iw->refs); /* claim ownership */ if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK) return false; spin_lock_irq(&iw->head->lock); list_del_init(&iwa->wo.child_wait.entry); spin_unlock_irq(&iw->head->lock); io_waitid_complete(req, -ECANCELED); return true; } int io_waitid_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd, unsigned int issue_flags) { struct hlist_node *tmp; struct io_kiocb *req; int nr = 0; if (cd->flags & (IORING_ASYNC_CANCEL_FD|IORING_ASYNC_CANCEL_FD_FIXED)) return -ENOENT; io_ring_submit_lock(ctx, issue_flags); hlist_for_each_entry_safe(req, tmp, &ctx->waitid_list, hash_node) { if (req->cqe.user_data != cd->data && !(cd->flags & IORING_ASYNC_CANCEL_ANY)) continue; if (__io_waitid_cancel(ctx, req)) nr++; if (!(cd->flags & IORING_ASYNC_CANCEL_ALL)) break; } io_ring_submit_unlock(ctx, issue_flags); if (nr) return nr; return -ENOENT; } bool io_waitid_remove_all(struct io_ring_ctx *ctx, struct io_uring_task *tctx, bool cancel_all) { struct hlist_node *tmp; struct io_kiocb *req; bool found = false; lockdep_assert_held(&ctx->uring_lock); hlist_for_each_entry_safe(req, tmp, &ctx->waitid_list, hash_node) { if (!io_match_task_safe(req, tctx, cancel_all)) continue; hlist_del_init(&req->hash_node); __io_waitid_cancel(ctx, req); found = true; } return found; } static inline bool io_waitid_drop_issue_ref(struct io_kiocb *req) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); struct io_waitid_async *iwa = req->async_data; if (!atomic_sub_return(1, &iw->refs)) return false; /* * Wakeup triggered, racing with us. It was prevented from * completing because of that, queue up the tw to do that. */ req->io_task_work.func = io_waitid_cb; io_req_task_work_add(req); remove_wait_queue(iw->head, &iwa->wo.child_wait); return true; } static void io_waitid_cb(struct io_kiocb *req, struct io_tw_state *ts) { struct io_waitid_async *iwa = req->async_data; struct io_ring_ctx *ctx = req->ctx; int ret; io_tw_lock(ctx, ts); ret = __do_wait(&iwa->wo); /* * If we get -ERESTARTSYS here, we need to re-arm and check again * to ensure we get another callback. If the retry works, then we can * just remove ourselves from the waitqueue again and finish the * request. */ if (unlikely(ret == -ERESTARTSYS)) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); /* Don't retry if cancel found it meanwhile */ ret = -ECANCELED; if (!(atomic_read(&iw->refs) & IO_WAITID_CANCEL_FLAG)) { iw->head = ¤t->signal->wait_chldexit; add_wait_queue(iw->head, &iwa->wo.child_wait); ret = __do_wait(&iwa->wo); if (ret == -ERESTARTSYS) { /* retry armed, drop our ref */ io_waitid_drop_issue_ref(req); return; } remove_wait_queue(iw->head, &iwa->wo.child_wait); } } io_waitid_complete(req, ret); } static int io_waitid_wait(struct wait_queue_entry *wait, unsigned mode, int sync, void *key) { struct wait_opts *wo = container_of(wait, struct wait_opts, child_wait); struct io_waitid_async *iwa = container_of(wo, struct io_waitid_async, wo); struct io_kiocb *req = iwa->req; struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); struct task_struct *p = key; if (!pid_child_should_wake(wo, p)) return 0; /* cancel is in progress */ if (atomic_fetch_inc(&iw->refs) & IO_WAITID_REF_MASK) return 1; req->io_task_work.func = io_waitid_cb; io_req_task_work_add(req); list_del_init(&wait->entry); return 1; } int io_waitid_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); if (sqe->addr || sqe->buf_index || sqe->addr3 || sqe->waitid_flags) return -EINVAL; iw->which = READ_ONCE(sqe->len); iw->upid = READ_ONCE(sqe->fd); iw->options = READ_ONCE(sqe->file_index); iw->infop = u64_to_user_ptr(READ_ONCE(sqe->addr2)); return 0; } int io_waitid(struct io_kiocb *req, unsigned int issue_flags) { struct io_waitid *iw = io_kiocb_to_cmd(req, struct io_waitid); struct io_ring_ctx *ctx = req->ctx; struct io_waitid_async *iwa; int ret; if (io_alloc_async_data(req)) return -ENOMEM; iwa = req->async_data; iwa->req = req; ret = kernel_waitid_prepare(&iwa->wo, iw->which, iw->upid, &iw->info, iw->options, NULL); if (ret) goto done; /* * Mark the request as busy upfront, in case we're racing with the * wakeup. If we are, then we'll notice when we drop this initial * reference again after arming. */ atomic_set(&iw->refs, 1); /* * Cancel must hold the ctx lock, so there's no risk of cancelation * finding us until a) we remain on the list, and b) the lock is * dropped. We only need to worry about racing with the wakeup * callback. */ io_ring_submit_lock(ctx, issue_flags); hlist_add_head(&req->hash_node, &ctx->waitid_list); init_waitqueue_func_entry(&iwa->wo.child_wait, io_waitid_wait); iwa->wo.child_wait.private = req->tctx->task; iw->head = ¤t->signal->wait_chldexit; add_wait_queue(iw->head, &iwa->wo.child_wait); ret = __do_wait(&iwa->wo); if (ret == -ERESTARTSYS) { /* * Nobody else grabbed a reference, it'll complete when we get * a waitqueue callback, or if someone cancels it. */ if (!io_waitid_drop_issue_ref(req)) { io_ring_submit_unlock(ctx, issue_flags); return IOU_ISSUE_SKIP_COMPLETE; } /* * Wakeup triggered, racing with us. It was prevented from * completing because of that, queue up the tw to do that. */ io_ring_submit_unlock(ctx, issue_flags); return IOU_ISSUE_SKIP_COMPLETE; } hlist_del_init(&req->hash_node); remove_wait_queue(iw->head, &iwa->wo.child_wait); ret = io_waitid_finish(req, ret); io_ring_submit_unlock(ctx, issue_flags); done: if (ret < 0) req_set_fail(req); io_req_set_res(req, ret, 0); return IOU_OK; } |
| 49 49 49 49 49 24 24 2 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 | /* FUSE: Filesystem in Userspace Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu> This program can be distributed under the terms of the GNU GPL. See the file COPYING. */ #include "fuse_i.h" #include <linux/init.h> #include <linux/module.h> #include <linux/fs_context.h> #define FUSE_CTL_SUPER_MAGIC 0x65735543 /* * This is non-NULL when the single instance of the control filesystem * exists. Protected by fuse_mutex */ static struct super_block *fuse_control_sb; static struct fuse_conn *fuse_ctl_file_conn_get(struct file *file) { struct fuse_conn *fc; mutex_lock(&fuse_mutex); fc = file_inode(file)->i_private; if (fc) fc = fuse_conn_get(fc); mutex_unlock(&fuse_mutex); return fc; } static ssize_t fuse_conn_abort_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { struct fuse_conn *fc = fuse_ctl_file_conn_get(file); if (fc) { if (fc->abort_err) fc->aborted = true; fuse_abort_conn(fc); fuse_conn_put(fc); } return count; } static ssize_t fuse_conn_waiting_read(struct file *file, char __user *buf, size_t len, loff_t *ppos) { char tmp[32]; size_t size; if (!*ppos) { long value; struct fuse_conn *fc = fuse_ctl_file_conn_get(file); if (!fc) return 0; value = atomic_read(&fc->num_waiting); file->private_data = (void *)value; fuse_conn_put(fc); } size = sprintf(tmp, "%ld\n", (long)file->private_data); return simple_read_from_buffer(buf, len, ppos, tmp, size); } static ssize_t fuse_conn_limit_read(struct file *file, char __user *buf, size_t len, loff_t *ppos, unsigned val) { char tmp[32]; size_t size = sprintf(tmp, "%u\n", val); return simple_read_from_buffer(buf, len, ppos, tmp, size); } static ssize_t fuse_conn_limit_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos, unsigned *val, unsigned global_limit) { unsigned long t; unsigned limit = (1 << 16) - 1; int err; if (*ppos) return -EINVAL; err = kstrtoul_from_user(buf, count, 0, &t); if (err) return err; if (!capable(CAP_SYS_ADMIN)) limit = min(limit, global_limit); if (t > limit) return -EINVAL; *val = t; return count; } static ssize_t fuse_conn_max_background_read(struct file *file, char __user *buf, size_t len, loff_t *ppos) { struct fuse_conn *fc; unsigned val; fc = fuse_ctl_file_conn_get(file); if (!fc) return 0; val = READ_ONCE(fc->max_background); fuse_conn_put(fc); return fuse_conn_limit_read(file, buf, len, ppos, val); } static ssize_t fuse_conn_max_background_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { unsigned val; ssize_t ret; ret = fuse_conn_limit_write(file, buf, count, ppos, &val, max_user_bgreq); if (ret > 0) { struct fuse_conn *fc = fuse_ctl_file_conn_get(file); if (fc) { spin_lock(&fc->bg_lock); fc->max_background = val; fc->blocked = fc->num_background >= fc->max_background; if (!fc->blocked) wake_up(&fc->blocked_waitq); spin_unlock(&fc->bg_lock); fuse_conn_put(fc); } } return ret; } static ssize_t fuse_conn_congestion_threshold_read(struct file *file, char __user *buf, size_t len, loff_t *ppos) { struct fuse_conn *fc; unsigned val; fc = fuse_ctl_file_conn_get(file); if (!fc) return 0; val = READ_ONCE(fc->congestion_threshold); fuse_conn_put(fc); return fuse_conn_limit_read(file, buf, len, ppos, val); } static ssize_t fuse_conn_congestion_threshold_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { unsigned val; struct fuse_conn *fc; ssize_t ret; ret = fuse_conn_limit_write(file, buf, count, ppos, &val, max_user_congthresh); if (ret <= 0) goto out; fc = fuse_ctl_file_conn_get(file); if (!fc) goto out; WRITE_ONCE(fc->congestion_threshold, val); fuse_conn_put(fc); out: return ret; } static const struct file_operations fuse_ctl_abort_ops = { .open = nonseekable_open, .write = fuse_conn_abort_write, }; static const struct file_operations fuse_ctl_waiting_ops = { .open = nonseekable_open, .read = fuse_conn_waiting_read, }; static const struct file_operations fuse_conn_max_background_ops = { .open = nonseekable_open, .read = fuse_conn_max_background_read, .write = fuse_conn_max_background_write, }; static const struct file_operations fuse_conn_congestion_threshold_ops = { .open = nonseekable_open, .read = fuse_conn_congestion_threshold_read, .write = fuse_conn_congestion_threshold_write, }; static struct dentry *fuse_ctl_add_dentry(struct dentry *parent, struct fuse_conn *fc, const char *name, int mode, int nlink, const struct inode_operations *iop, const struct file_operations *fop) { struct dentry *dentry; struct inode *inode; BUG_ON(fc->ctl_ndents >= FUSE_CTL_NUM_DENTRIES); dentry = d_alloc_name(parent, name); if (!dentry) return NULL; inode = new_inode(fuse_control_sb); if (!inode) { dput(dentry); return NULL; } inode->i_ino = get_next_ino(); inode->i_mode = mode; inode->i_uid = fc->user_id; inode->i_gid = fc->group_id; simple_inode_init_ts(inode); /* setting ->i_op to NULL is not allowed */ if (iop) inode->i_op = iop; inode->i_fop = fop; set_nlink(inode, nlink); inode->i_private = fc; d_add(dentry, inode); fc->ctl_dentry[fc->ctl_ndents++] = dentry; return dentry; } /* * Add a connection to the control filesystem (if it exists). Caller * must hold fuse_mutex */ int fuse_ctl_add_conn(struct fuse_conn *fc) { struct dentry *parent; char name[32]; if (!fuse_control_sb || fc->no_control) return 0; parent = fuse_control_sb->s_root; inc_nlink(d_inode(parent)); sprintf(name, "%u", fc->dev); parent = fuse_ctl_add_dentry(parent, fc, name, S_IFDIR | 0500, 2, &simple_dir_inode_operations, &simple_dir_operations); if (!parent) goto err; if (!fuse_ctl_add_dentry(parent, fc, "waiting", S_IFREG | 0400, 1, NULL, &fuse_ctl_waiting_ops) || !fuse_ctl_add_dentry(parent, fc, "abort", S_IFREG | 0200, 1, NULL, &fuse_ctl_abort_ops) || !fuse_ctl_add_dentry(parent, fc, "max_background", S_IFREG | 0600, 1, NULL, &fuse_conn_max_background_ops) || !fuse_ctl_add_dentry(parent, fc, "congestion_threshold", S_IFREG | 0600, 1, NULL, &fuse_conn_congestion_threshold_ops)) goto err; return 0; err: fuse_ctl_remove_conn(fc); return -ENOMEM; } /* * Remove a connection from the control filesystem (if it exists). * Caller must hold fuse_mutex */ void fuse_ctl_remove_conn(struct fuse_conn *fc) { int i; if (!fuse_control_sb || fc->no_control) return; for (i = fc->ctl_ndents - 1; i >= 0; i--) { struct dentry *dentry = fc->ctl_dentry[i]; d_inode(dentry)->i_private = NULL; if (!i) { /* Get rid of submounts: */ d_invalidate(dentry); } dput(dentry); } drop_nlink(d_inode(fuse_control_sb->s_root)); } static int fuse_ctl_fill_super(struct super_block *sb, struct fs_context *fsc) { static const struct tree_descr empty_descr = {""}; struct fuse_conn *fc; int err; err = simple_fill_super(sb, FUSE_CTL_SUPER_MAGIC, &empty_descr); if (err) return err; mutex_lock(&fuse_mutex); BUG_ON(fuse_control_sb); fuse_control_sb = sb; list_for_each_entry(fc, &fuse_conn_list, entry) { err = fuse_ctl_add_conn(fc); if (err) { fuse_control_sb = NULL; mutex_unlock(&fuse_mutex); return err; } } mutex_unlock(&fuse_mutex); return 0; } static int fuse_ctl_get_tree(struct fs_context *fsc) { return get_tree_single(fsc, fuse_ctl_fill_super); } static const struct fs_context_operations fuse_ctl_context_ops = { .get_tree = fuse_ctl_get_tree, }; static int fuse_ctl_init_fs_context(struct fs_context *fsc) { fsc->ops = &fuse_ctl_context_ops; return 0; } static void fuse_ctl_kill_sb(struct super_block *sb) { struct fuse_conn *fc; mutex_lock(&fuse_mutex); fuse_control_sb = NULL; list_for_each_entry(fc, &fuse_conn_list, entry) fc->ctl_ndents = 0; mutex_unlock(&fuse_mutex); kill_litter_super(sb); } static struct file_system_type fuse_ctl_fs_type = { .owner = THIS_MODULE, .name = "fusectl", .init_fs_context = fuse_ctl_init_fs_context, .kill_sb = fuse_ctl_kill_sb, }; MODULE_ALIAS_FS("fusectl"); int __init fuse_ctl_init(void) { return register_filesystem(&fuse_ctl_fs_type); } void __exit fuse_ctl_cleanup(void) { unregister_filesystem(&fuse_ctl_fs_type); } |
| 23 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_X86_UNWIND_H #define _ASM_X86_UNWIND_H #include <linux/sched.h> #include <linux/ftrace.h> #include <linux/rethook.h> #include <asm/ptrace.h> #include <asm/stacktrace.h> #define IRET_FRAME_OFFSET (offsetof(struct pt_regs, ip)) #define IRET_FRAME_SIZE (sizeof(struct pt_regs) - IRET_FRAME_OFFSET) struct unwind_state { struct stack_info stack_info; unsigned long stack_mask; struct task_struct *task; int graph_idx; #if defined(CONFIG_RETHOOK) struct llist_node *kr_cur; #endif bool error; #if defined(CONFIG_UNWINDER_ORC) bool signal, full_regs; unsigned long sp, bp, ip; struct pt_regs *regs, *prev_regs; #elif defined(CONFIG_UNWINDER_FRAME_POINTER) bool got_irq; unsigned long *bp, *orig_sp, ip; /* * If non-NULL: The current frame is incomplete and doesn't contain a * valid BP. When looking for the next frame, use this instead of the * non-existent saved BP. */ unsigned long *next_bp; struct pt_regs *regs; #else unsigned long *sp; #endif }; void __unwind_start(struct unwind_state *state, struct task_struct *task, struct pt_regs *regs, unsigned long *first_frame); bool unwind_next_frame(struct unwind_state *state); unsigned long unwind_get_return_address(struct unwind_state *state); unsigned long *unwind_get_return_address_ptr(struct unwind_state *state); static inline bool unwind_done(struct unwind_state *state) { return state->stack_info.type == STACK_TYPE_UNKNOWN; } static inline bool unwind_error(struct unwind_state *state) { return state->error; } static inline void unwind_start(struct unwind_state *state, struct task_struct *task, struct pt_regs *regs, unsigned long *first_frame) { first_frame = first_frame ? : get_stack_pointer(task, regs); __unwind_start(state, task, regs, first_frame); } #if defined(CONFIG_UNWINDER_ORC) || defined(CONFIG_UNWINDER_FRAME_POINTER) /* * If 'partial' returns true, only the iret frame registers are valid. */ static inline struct pt_regs *unwind_get_entry_regs(struct unwind_state *state, bool *partial) { if (unwind_done(state)) return NULL; if (partial) { #ifdef CONFIG_UNWINDER_ORC *partial = !state->full_regs; #else *partial = false; #endif } return state->regs; } #else static inline struct pt_regs *unwind_get_entry_regs(struct unwind_state *state, bool *partial) { return NULL; } #endif #ifdef CONFIG_UNWINDER_ORC void unwind_init(void); void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size); #else static inline void unwind_init(void) {} static inline void unwind_module_init(struct module *mod, void *orc_ip, size_t orc_ip_size, void *orc, size_t orc_size) {} #endif static inline unsigned long unwind_recover_rethook(struct unwind_state *state, unsigned long addr, unsigned long *addr_p) { #ifdef CONFIG_RETHOOK if (is_rethook_trampoline(addr)) return rethook_find_ret_addr(state->task, (unsigned long)addr_p, &state->kr_cur); #endif return addr; } /* Recover the return address modified by rethook and ftrace_graph. */ static inline unsigned long unwind_recover_ret_addr(struct unwind_state *state, unsigned long addr, unsigned long *addr_p) { unsigned long ret; ret = ftrace_graph_ret_addr(state->task, &state->graph_idx, addr, addr_p); return unwind_recover_rethook(state, ret, addr_p); } /* * This disables KASAN checking when reading a value from another task's stack, * since the other task could be running on another CPU and could have poisoned * the stack in the meantime. */ #define READ_ONCE_TASK_STACK(task, x) \ ({ \ unsigned long val; \ if (task == current) \ val = READ_ONCE(x); \ else \ val = READ_ONCE_NOCHECK(x); \ val; \ }) static inline bool task_on_another_cpu(struct task_struct *task) { #ifdef CONFIG_SMP return task != current && task->on_cpu; #else return false; #endif } #endif /* _ASM_X86_UNWIND_H */ |
| 94 94 93 100 92 94 94 93 79 80 31 31 30 31 10 38 80 79 78 80 80 80 80 80 79 80 38 80 78 79 103 103 103 5 75 70 69 69 70 69 70 74 1 3 78 79 78 1 3 75 76 79 79 76 10 103 4 4 4 1 4 4 103 64 37 63 100 101 101 101 55 103 103 103 5 101 75 73 77 25 25 102 66 102 103 101 5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 | /* SPDX-License-Identifier: GPL-2.0 * * page_pool.c * Author: Jesper Dangaard Brouer <netoptimizer@brouer.com> * Copyright (C) 2016 Red Hat, Inc. */ #include <linux/error-injection.h> #include <linux/types.h> #include <linux/kernel.h> #include <linux/slab.h> #include <linux/device.h> #include <net/netdev_rx_queue.h> #include <net/page_pool/helpers.h> #include <net/xdp.h> #include <linux/dma-direction.h> #include <linux/dma-mapping.h> #include <linux/page-flags.h> #include <linux/mm.h> /* for put_page() */ #include <linux/poison.h> #include <linux/ethtool.h> #include <linux/netdevice.h> #include <trace/events/page_pool.h> #include "mp_dmabuf_devmem.h" #include "netmem_priv.h" #include "page_pool_priv.h" DEFINE_STATIC_KEY_FALSE(page_pool_mem_providers); #define DEFER_TIME (msecs_to_jiffies(1000)) #define DEFER_WARN_INTERVAL (60 * HZ) #define BIAS_MAX (LONG_MAX >> 1) #ifdef CONFIG_PAGE_POOL_STATS static DEFINE_PER_CPU(struct page_pool_recycle_stats, pp_system_recycle_stats); /* alloc_stat_inc is intended to be used in softirq context */ #define alloc_stat_inc(pool, __stat) (pool->alloc_stats.__stat++) /* recycle_stat_inc is safe to use when preemption is possible. */ #define recycle_stat_inc(pool, __stat) \ do { \ struct page_pool_recycle_stats __percpu *s = pool->recycle_stats; \ this_cpu_inc(s->__stat); \ } while (0) #define recycle_stat_add(pool, __stat, val) \ do { \ struct page_pool_recycle_stats __percpu *s = pool->recycle_stats; \ this_cpu_add(s->__stat, val); \ } while (0) static const char pp_stats[][ETH_GSTRING_LEN] = { "rx_pp_alloc_fast", "rx_pp_alloc_slow", "rx_pp_alloc_slow_ho", "rx_pp_alloc_empty", "rx_pp_alloc_refill", "rx_pp_alloc_waive", "rx_pp_recycle_cached", "rx_pp_recycle_cache_full", "rx_pp_recycle_ring", "rx_pp_recycle_ring_full", "rx_pp_recycle_released_ref", }; /** * page_pool_get_stats() - fetch page pool stats * @pool: pool from which page was allocated * @stats: struct page_pool_stats to fill in * * Retrieve statistics about the page_pool. This API is only available * if the kernel has been configured with ``CONFIG_PAGE_POOL_STATS=y``. * A pointer to a caller allocated struct page_pool_stats structure * is passed to this API which is filled in. The caller can then report * those stats to the user (perhaps via ethtool, debugfs, etc.). */ bool page_pool_get_stats(const struct page_pool *pool, struct page_pool_stats *stats) { int cpu = 0; if (!stats) return false; /* The caller is responsible to initialize stats. */ stats->alloc_stats.fast += pool->alloc_stats.fast; stats->alloc_stats.slow += pool->alloc_stats.slow; stats->alloc_stats.slow_high_order += pool->alloc_stats.slow_high_order; stats->alloc_stats.empty += pool->alloc_stats.empty; stats->alloc_stats.refill += pool->alloc_stats.refill; stats->alloc_stats.waive += pool->alloc_stats.waive; for_each_possible_cpu(cpu) { const struct page_pool_recycle_stats *pcpu = per_cpu_ptr(pool->recycle_stats, cpu); stats->recycle_stats.cached += pcpu->cached; stats->recycle_stats.cache_full += pcpu->cache_full; stats->recycle_stats.ring += pcpu->ring; stats->recycle_stats.ring_full += pcpu->ring_full; stats->recycle_stats.released_refcnt += pcpu->released_refcnt; } return true; } EXPORT_SYMBOL(page_pool_get_stats); u8 *page_pool_ethtool_stats_get_strings(u8 *data) { int i; for (i = 0; i < ARRAY_SIZE(pp_stats); i++) { memcpy(data, pp_stats[i], ETH_GSTRING_LEN); data += ETH_GSTRING_LEN; } return data; } EXPORT_SYMBOL(page_pool_ethtool_stats_get_strings); int page_pool_ethtool_stats_get_count(void) { return ARRAY_SIZE(pp_stats); } EXPORT_SYMBOL(page_pool_ethtool_stats_get_count); u64 *page_pool_ethtool_stats_get(u64 *data, const void *stats) { const struct page_pool_stats *pool_stats = stats; *data++ = pool_stats->alloc_stats.fast; *data++ = pool_stats->alloc_stats.slow; *data++ = pool_stats->alloc_stats.slow_high_order; *data++ = pool_stats->alloc_stats.empty; *data++ = pool_stats->alloc_stats.refill; *data++ = pool_stats->alloc_stats.waive; *data++ = pool_stats->recycle_stats.cached; *data++ = pool_stats->recycle_stats.cache_full; *data++ = pool_stats->recycle_stats.ring; *data++ = pool_stats->recycle_stats.ring_full; *data++ = pool_stats->recycle_stats.released_refcnt; return data; } EXPORT_SYMBOL(page_pool_ethtool_stats_get); #else #define alloc_stat_inc(pool, __stat) #define recycle_stat_inc(pool, __stat) #define recycle_stat_add(pool, __stat, val) #endif static bool page_pool_producer_lock(struct page_pool *pool) __acquires(&pool->ring.producer_lock) { bool in_softirq = in_softirq(); if (in_softirq) spin_lock(&pool->ring.producer_lock); else spin_lock_bh(&pool->ring.producer_lock); return in_softirq; } static void page_pool_producer_unlock(struct page_pool *pool, bool in_softirq) __releases(&pool->ring.producer_lock) { if (in_softirq) spin_unlock(&pool->ring.producer_lock); else spin_unlock_bh(&pool->ring.producer_lock); } static void page_pool_struct_check(void) { CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_users); CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_page); CACHELINE_ASSERT_GROUP_MEMBER(struct page_pool, frag, frag_offset); CACHELINE_ASSERT_GROUP_SIZE(struct page_pool, frag, PAGE_POOL_FRAG_GROUP_ALIGN); } static int page_pool_init(struct page_pool *pool, const struct page_pool_params *params, int cpuid) { unsigned int ring_qsize = 1024; /* Default */ struct netdev_rx_queue *rxq; int err; page_pool_struct_check(); memcpy(&pool->p, ¶ms->fast, sizeof(pool->p)); memcpy(&pool->slow, ¶ms->slow, sizeof(pool->slow)); pool->cpuid = cpuid; /* Validate only known flags were used */ if (pool->slow.flags & ~PP_FLAG_ALL) return -EINVAL; if (pool->p.pool_size) ring_qsize = pool->p.pool_size; /* Sanity limit mem that can be pinned down */ if (ring_qsize > 32768) return -E2BIG; /* DMA direction is either DMA_FROM_DEVICE or DMA_BIDIRECTIONAL. * DMA_BIDIRECTIONAL is for allowing page used for DMA sending, * which is the XDP_TX use-case. */ if (pool->slow.flags & PP_FLAG_DMA_MAP) { if ((pool->p.dma_dir != DMA_FROM_DEVICE) && (pool->p.dma_dir != DMA_BIDIRECTIONAL)) return -EINVAL; pool->dma_map = true; } if (pool->slow.flags & PP_FLAG_DMA_SYNC_DEV) { /* In order to request DMA-sync-for-device the page * needs to be mapped */ if (!(pool->slow.flags & PP_FLAG_DMA_MAP)) return -EINVAL; if (!pool->p.max_len) return -EINVAL; pool->dma_sync = true; /* pool->p.offset has to be set according to the address * offset used by the DMA engine to start copying rx data */ } pool->has_init_callback = !!pool->slow.init_callback; #ifdef CONFIG_PAGE_POOL_STATS if (!(pool->slow.flags & PP_FLAG_SYSTEM_POOL)) { pool->recycle_stats = alloc_percpu(struct page_pool_recycle_stats); if (!pool->recycle_stats) return -ENOMEM; } else { /* For system page pool instance we use a singular stats object * instead of allocating a separate percpu variable for each * (also percpu) page pool instance. */ pool->recycle_stats = &pp_system_recycle_stats; pool->system = true; } #endif if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0) { #ifdef CONFIG_PAGE_POOL_STATS if (!pool->system) free_percpu(pool->recycle_stats); #endif return -ENOMEM; } atomic_set(&pool->pages_state_release_cnt, 0); /* Driver calling page_pool_create() also call page_pool_destroy() */ refcount_set(&pool->user_cnt, 1); if (pool->dma_map) get_device(pool->p.dev); if (pool->slow.flags & PP_FLAG_ALLOW_UNREADABLE_NETMEM) { /* We rely on rtnl_lock()ing to make sure netdev_rx_queue * configuration doesn't change while we're initializing * the page_pool. */ ASSERT_RTNL(); rxq = __netif_get_rx_queue(pool->slow.netdev, pool->slow.queue_idx); pool->mp_priv = rxq->mp_params.mp_priv; } if (pool->mp_priv) { err = mp_dmabuf_devmem_init(pool); if (err) { pr_warn("%s() mem-provider init failed %d\n", __func__, err); goto free_ptr_ring; } static_branch_inc(&page_pool_mem_providers); } return 0; free_ptr_ring: ptr_ring_cleanup(&pool->ring, NULL); #ifdef CONFIG_PAGE_POOL_STATS if (!pool->system) free_percpu(pool->recycle_stats); #endif return err; } static void page_pool_uninit(struct page_pool *pool) { ptr_ring_cleanup(&pool->ring, NULL); if (pool->dma_map) put_device(pool->p.dev); #ifdef CONFIG_PAGE_POOL_STATS if (!pool->system) free_percpu(pool->recycle_stats); #endif } /** * page_pool_create_percpu() - create a page pool for a given cpu. * @params: parameters, see struct page_pool_params * @cpuid: cpu identifier */ struct page_pool * page_pool_create_percpu(const struct page_pool_params *params, int cpuid) { struct page_pool *pool; int err; pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid); if (!pool) return ERR_PTR(-ENOMEM); err = page_pool_init(pool, params, cpuid); if (err < 0) goto err_free; err = page_pool_list(pool); if (err) goto err_uninit; return pool; err_uninit: page_pool_uninit(pool); err_free: pr_warn("%s() gave up with errno %d\n", __func__, err); kfree(pool); return ERR_PTR(err); } EXPORT_SYMBOL(page_pool_create_percpu); /** * page_pool_create() - create a page pool * @params: parameters, see struct page_pool_params */ struct page_pool *page_pool_create(const struct page_pool_params *params) { return page_pool_create_percpu(params, -1); } EXPORT_SYMBOL(page_pool_create); static void page_pool_return_page(struct page_pool *pool, netmem_ref netmem); static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool) { struct ptr_ring *r = &pool->ring; netmem_ref netmem; int pref_nid; /* preferred NUMA node */ /* Quicker fallback, avoid locks when ring is empty */ if (__ptr_ring_empty(r)) { alloc_stat_inc(pool, empty); return 0; } /* Softirq guarantee CPU and thus NUMA node is stable. This, * assumes CPU refilling driver RX-ring will also run RX-NAPI. */ #ifdef CONFIG_NUMA pref_nid = (pool->p.nid == NUMA_NO_NODE) ? numa_mem_id() : pool->p.nid; #else /* Ignore pool->p.nid setting if !CONFIG_NUMA, helps compiler */ pref_nid = numa_mem_id(); /* will be zero like page_to_nid() */ #endif /* Refill alloc array, but only if NUMA match */ do { netmem = (__force netmem_ref)__ptr_ring_consume(r); if (unlikely(!netmem)) break; if (likely(netmem_is_pref_nid(netmem, pref_nid))) { pool->alloc.cache[pool->alloc.count++] = netmem; } else { /* NUMA mismatch; * (1) release 1 page to page-allocator and * (2) break out to fallthrough to alloc_pages_node. * This limit stress on page buddy alloactor. */ page_pool_return_page(pool, netmem); alloc_stat_inc(pool, waive); netmem = 0; break; } } while (pool->alloc.count < PP_ALLOC_CACHE_REFILL); /* Return last page */ if (likely(pool->alloc.count > 0)) { netmem = pool->alloc.cache[--pool->alloc.count]; alloc_stat_inc(pool, refill); } return netmem; } /* fast path */ static netmem_ref __page_pool_get_cached(struct page_pool *pool) { netmem_ref netmem; /* Caller MUST guarantee safe non-concurrent access, e.g. softirq */ if (likely(pool->alloc.count)) { /* Fast-path */ netmem = pool->alloc.cache[--pool->alloc.count]; alloc_stat_inc(pool, fast); } else { netmem = page_pool_refill_alloc_cache(pool); } return netmem; } static void __page_pool_dma_sync_for_device(const struct page_pool *pool, netmem_ref netmem, u32 dma_sync_size) { #if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC) dma_addr_t dma_addr = page_pool_get_dma_addr_netmem(netmem); dma_sync_size = min(dma_sync_size, pool->p.max_len); __dma_sync_single_for_device(pool->p.dev, dma_addr + pool->p.offset, dma_sync_size, pool->p.dma_dir); #endif } static __always_inline void page_pool_dma_sync_for_device(const struct page_pool *pool, netmem_ref netmem, u32 dma_sync_size) { if (pool->dma_sync && dma_dev_need_sync(pool->p.dev)) __page_pool_dma_sync_for_device(pool, netmem, dma_sync_size); } static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem) { dma_addr_t dma; /* Setup DMA mapping: use 'struct page' area for storing DMA-addr * since dma_addr_t can be either 32 or 64 bits and does not always fit * into page private data (i.e 32bit cpu with 64bit DMA caps) * This mapping is kept for lifetime of page, until leaving pool. */ dma = dma_map_page_attrs(pool->p.dev, netmem_to_page(netmem), 0, (PAGE_SIZE << pool->p.order), pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); if (dma_mapping_error(pool->p.dev, dma)) return false; if (page_pool_set_dma_addr_netmem(netmem, dma)) goto unmap_failed; page_pool_dma_sync_for_device(pool, netmem, pool->p.max_len); return true; unmap_failed: WARN_ONCE(1, "unexpected DMA address, please report to netdev@"); dma_unmap_page_attrs(pool->p.dev, dma, PAGE_SIZE << pool->p.order, pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); return false; } static struct page *__page_pool_alloc_page_order(struct page_pool *pool, gfp_t gfp) { struct page *page; gfp |= __GFP_COMP; page = alloc_pages_node(pool->p.nid, gfp, pool->p.order); if (unlikely(!page)) return NULL; if (pool->dma_map && unlikely(!page_pool_dma_map(pool, page_to_netmem(page)))) { put_page(page); return NULL; } alloc_stat_inc(pool, slow_high_order); page_pool_set_pp_info(pool, page_to_netmem(page)); /* Track how many pages are held 'in-flight' */ pool->pages_state_hold_cnt++; trace_page_pool_state_hold(pool, page_to_netmem(page), pool->pages_state_hold_cnt); return page; } /* slow path */ static noinline netmem_ref __page_pool_alloc_pages_slow(struct page_pool *pool, gfp_t gfp) { const int bulk = PP_ALLOC_CACHE_REFILL; unsigned int pp_order = pool->p.order; bool dma_map = pool->dma_map; netmem_ref netmem; int i, nr_pages; /* Don't support bulk alloc for high-order pages */ if (unlikely(pp_order)) return page_to_netmem(__page_pool_alloc_page_order(pool, gfp)); /* Unnecessary as alloc cache is empty, but guarantees zero count */ if (unlikely(pool->alloc.count > 0)) return pool->alloc.cache[--pool->alloc.count]; /* Mark empty alloc.cache slots "empty" for alloc_pages_bulk_array */ memset(&pool->alloc.cache, 0, sizeof(void *) * bulk); nr_pages = alloc_pages_bulk_array_node(gfp, pool->p.nid, bulk, (struct page **)pool->alloc.cache); if (unlikely(!nr_pages)) return 0; /* Pages have been filled into alloc.cache array, but count is zero and * page element have not been (possibly) DMA mapped. */ for (i = 0; i < nr_pages; i++) { netmem = pool->alloc.cache[i]; if (dma_map && unlikely(!page_pool_dma_map(pool, netmem))) { put_page(netmem_to_page(netmem)); continue; } page_pool_set_pp_info(pool, netmem); pool->alloc.cache[pool->alloc.count++] = netmem; /* Track how many pages are held 'in-flight' */ pool->pages_state_hold_cnt++; trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt); } /* Return last page */ if (likely(pool->alloc.count > 0)) { netmem = pool->alloc.cache[--pool->alloc.count]; alloc_stat_inc(pool, slow); } else { netmem = 0; } /* When page just alloc'ed is should/must have refcnt 1. */ return netmem; } /* For using page_pool replace: alloc_pages() API calls, but provide * synchronization guarantee for allocation side. */ netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp) { netmem_ref netmem; /* Fast-path: Get a page from cache */ netmem = __page_pool_get_cached(pool); if (netmem) return netmem; /* Slow-path: cache empty, do real allocation */ if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_priv) netmem = mp_dmabuf_devmem_alloc_netmems(pool, gfp); else netmem = __page_pool_alloc_pages_slow(pool, gfp); return netmem; } EXPORT_SYMBOL(page_pool_alloc_netmem); struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp) { return netmem_to_page(page_pool_alloc_netmem(pool, gfp)); } EXPORT_SYMBOL(page_pool_alloc_pages); ALLOW_ERROR_INJECTION(page_pool_alloc_pages, NULL); /* Calculate distance between two u32 values, valid if distance is below 2^(31) * https://en.wikipedia.org/wiki/Serial_number_arithmetic#General_Solution */ #define _distance(a, b) (s32)((a) - (b)) s32 page_pool_inflight(const struct page_pool *pool, bool strict) { u32 release_cnt = atomic_read(&pool->pages_state_release_cnt); u32 hold_cnt = READ_ONCE(pool->pages_state_hold_cnt); s32 inflight; inflight = _distance(hold_cnt, release_cnt); if (strict) { trace_page_pool_release(pool, inflight, hold_cnt, release_cnt); WARN(inflight < 0, "Negative(%d) inflight packet-pages", inflight); } else { inflight = max(0, inflight); } return inflight; } void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem) { netmem_set_pp(netmem, pool); netmem_or_pp_magic(netmem, PP_SIGNATURE); /* Ensuring all pages have been split into one fragment initially: * page_pool_set_pp_info() is only called once for every page when it * is allocated from the page allocator and page_pool_fragment_page() * is dirtying the same cache line as the page->pp_magic above, so * the overhead is negligible. */ page_pool_fragment_netmem(netmem, 1); if (pool->has_init_callback) pool->slow.init_callback(netmem, pool->slow.init_arg); } void page_pool_clear_pp_info(netmem_ref netmem) { netmem_clear_pp_magic(netmem); netmem_set_pp(netmem, NULL); } static __always_inline void __page_pool_release_page_dma(struct page_pool *pool, netmem_ref netmem) { dma_addr_t dma; if (!pool->dma_map) /* Always account for inflight pages, even if we didn't * map them */ return; dma = page_pool_get_dma_addr_netmem(netmem); /* When page is unmapped, it cannot be returned to our pool */ dma_unmap_page_attrs(pool->p.dev, dma, PAGE_SIZE << pool->p.order, pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); page_pool_set_dma_addr_netmem(netmem, 0); } /* Disconnects a page (from a page_pool). API users can have a need * to disconnect a page (from a page_pool), to allow it to be used as * a regular page (that will eventually be returned to the normal * page-allocator via put_page). */ void page_pool_return_page(struct page_pool *pool, netmem_ref netmem) { int count; bool put; put = true; if (static_branch_unlikely(&page_pool_mem_providers) && pool->mp_priv) put = mp_dmabuf_devmem_release_page(pool, netmem); else __page_pool_release_page_dma(pool, netmem); /* This may be the last page returned, releasing the pool, so * it is not safe to reference pool afterwards. */ count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt); trace_page_pool_state_release(pool, netmem, count); if (put) { page_pool_clear_pp_info(netmem); put_page(netmem_to_page(netmem)); } /* An optimization would be to call __free_pages(page, pool->p.order) * knowing page is not part of page-cache (thus avoiding a * __page_cache_release() call). */ } static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem) { int ret; /* BH protection not needed if current is softirq */ if (in_softirq()) ret = ptr_ring_produce(&pool->ring, (__force void *)netmem); else ret = ptr_ring_produce_bh(&pool->ring, (__force void *)netmem); if (!ret) { recycle_stat_inc(pool, ring); return true; } return false; } /* Only allow direct recycling in special circumstances, into the * alloc side cache. E.g. during RX-NAPI processing for XDP_DROP use-case. * * Caller must provide appropriate safe context. */ static bool page_pool_recycle_in_cache(netmem_ref netmem, struct page_pool *pool) { if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) { recycle_stat_inc(pool, cache_full); return false; } /* Caller MUST have verified/know (page_ref_count(page) == 1) */ pool->alloc.cache[pool->alloc.count++] = netmem; recycle_stat_inc(pool, cached); return true; } static bool __page_pool_page_can_be_recycled(netmem_ref netmem) { return netmem_is_net_iov(netmem) || (page_ref_count(netmem_to_page(netmem)) == 1 && !page_is_pfmemalloc(netmem_to_page(netmem))); } /* If the page refcnt == 1, this will try to recycle the page. * If pool->dma_sync is set, we'll try to sync the DMA area for * the configured size min(dma_sync_size, pool->max_len). * If the page refcnt != 1, then the page will be returned to memory * subsystem. */ static __always_inline netmem_ref __page_pool_put_page(struct page_pool *pool, netmem_ref netmem, unsigned int dma_sync_size, bool allow_direct) { lockdep_assert_no_hardirq(); /* This allocator is optimized for the XDP mode that uses * one-frame-per-page, but have fallbacks that act like the * regular page allocator APIs. * * refcnt == 1 means page_pool owns page, and can recycle it. * * page is NOT reusable when allocated when system is under * some pressure. (page_is_pfmemalloc) */ if (likely(__page_pool_page_can_be_recycled(netmem))) { /* Read barrier done in page_ref_count / READ_ONCE */ page_pool_dma_sync_for_device(pool, netmem, dma_sync_size); if (allow_direct && page_pool_recycle_in_cache(netmem, pool)) return 0; /* Page found as candidate for recycling */ return netmem; } /* Fallback/non-XDP mode: API user have elevated refcnt. * * Many drivers split up the page into fragments, and some * want to keep doing this to save memory and do refcnt based * recycling. Support this use case too, to ease drivers * switching between XDP/non-XDP. * * In-case page_pool maintains the DMA mapping, API user must * call page_pool_put_page once. In this elevated refcnt * case, the DMA is unmapped/released, as driver is likely * doing refcnt based recycle tricks, meaning another process * will be invoking put_page. */ recycle_stat_inc(pool, released_refcnt); page_pool_return_page(pool, netmem); return 0; } static bool page_pool_napi_local(const struct page_pool *pool) { const struct napi_struct *napi; u32 cpuid; if (unlikely(!in_softirq())) return false; /* Allow direct recycle if we have reasons to believe that we are * in the same context as the consumer would run, so there's * no possible race. * __page_pool_put_page() makes sure we're not in hardirq context * and interrupts are enabled prior to accessing the cache. */ cpuid = smp_processor_id(); if (READ_ONCE(pool->cpuid) == cpuid) return true; napi = READ_ONCE(pool->p.napi); return napi && READ_ONCE(napi->list_owner) == cpuid; } void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem, unsigned int dma_sync_size, bool allow_direct) { if (!allow_direct) allow_direct = page_pool_napi_local(pool); netmem = __page_pool_put_page(pool, netmem, dma_sync_size, allow_direct); if (netmem && !page_pool_recycle_in_ring(pool, netmem)) { /* Cache full, fallback to free pages */ recycle_stat_inc(pool, ring_full); page_pool_return_page(pool, netmem); } } EXPORT_SYMBOL(page_pool_put_unrefed_netmem); void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page, unsigned int dma_sync_size, bool allow_direct) { page_pool_put_unrefed_netmem(pool, page_to_netmem(page), dma_sync_size, allow_direct); } EXPORT_SYMBOL(page_pool_put_unrefed_page); /** * page_pool_put_page_bulk() - release references on multiple pages * @pool: pool from which pages were allocated * @data: array holding page pointers * @count: number of pages in @data * * Tries to refill a number of pages into the ptr_ring cache holding ptr_ring * producer lock. If the ptr_ring is full, page_pool_put_page_bulk() * will release leftover pages to the page allocator. * page_pool_put_page_bulk() is suitable to be run inside the driver NAPI tx * completion loop for the XDP_REDIRECT use case. * * Please note the caller must not use data area after running * page_pool_put_page_bulk(), as this function overwrites it. */ void page_pool_put_page_bulk(struct page_pool *pool, void **data, int count) { int i, bulk_len = 0; bool allow_direct; bool in_softirq; allow_direct = page_pool_napi_local(pool); for (i = 0; i < count; i++) { netmem_ref netmem = page_to_netmem(virt_to_head_page(data[i])); /* It is not the last user for the page frag case */ if (!page_pool_is_last_ref(netmem)) continue; netmem = __page_pool_put_page(pool, netmem, -1, allow_direct); /* Approved for bulk recycling in ptr_ring cache */ if (netmem) data[bulk_len++] = (__force void *)netmem; } if (!bulk_len) return; /* Bulk producer into ptr_ring page_pool cache */ in_softirq = page_pool_producer_lock(pool); for (i = 0; i < bulk_len; i++) { if (__ptr_ring_produce(&pool->ring, data[i])) { /* ring full */ recycle_stat_inc(pool, ring_full); break; } } recycle_stat_add(pool, ring, i); page_pool_producer_unlock(pool, in_softirq); /* Hopefully all pages was return into ptr_ring */ if (likely(i == bulk_len)) return; /* ptr_ring cache full, free remaining pages outside producer lock * since put_page() with refcnt == 1 can be an expensive operation */ for (; i < bulk_len; i++) page_pool_return_page(pool, (__force netmem_ref)data[i]); } EXPORT_SYMBOL(page_pool_put_page_bulk); static netmem_ref page_pool_drain_frag(struct page_pool *pool, netmem_ref netmem) { long drain_count = BIAS_MAX - pool->frag_users; /* Some user is still using the page frag */ if (likely(page_pool_unref_netmem(netmem, drain_count))) return 0; if (__page_pool_page_can_be_recycled(netmem)) { page_pool_dma_sync_for_device(pool, netmem, -1); return netmem; } page_pool_return_page(pool, netmem); return 0; } static void page_pool_free_frag(struct page_pool *pool) { long drain_count = BIAS_MAX - pool->frag_users; netmem_ref netmem = pool->frag_page; pool->frag_page = 0; if (!netmem || page_pool_unref_netmem(netmem, drain_count)) return; page_pool_return_page(pool, netmem); } netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool, unsigned int *offset, unsigned int size, gfp_t gfp) { unsigned int max_size = PAGE_SIZE << pool->p.order; netmem_ref netmem = pool->frag_page; if (WARN_ON(size > max_size)) return 0; size = ALIGN(size, dma_get_cache_alignment()); *offset = pool->frag_offset; if (netmem && *offset + size > max_size) { netmem = page_pool_drain_frag(pool, netmem); if (netmem) { recycle_stat_inc(pool, cached); alloc_stat_inc(pool, fast); goto frag_reset; } } if (!netmem) { netmem = page_pool_alloc_netmem(pool, gfp); if (unlikely(!netmem)) { pool->frag_page = 0; return 0; } pool->frag_page = netmem; frag_reset: pool->frag_users = 1; *offset = 0; pool->frag_offset = size; page_pool_fragment_netmem(netmem, BIAS_MAX); return netmem; } pool->frag_users++; pool->frag_offset = *offset + size; return netmem; } EXPORT_SYMBOL(page_pool_alloc_frag_netmem); struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, unsigned int size, gfp_t gfp) { return netmem_to_page(page_pool_alloc_frag_netmem(pool, offset, size, gfp)); } EXPORT_SYMBOL(page_pool_alloc_frag); static void page_pool_empty_ring(struct page_pool *pool) { netmem_ref netmem; /* Empty recycle ring */ while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) { /* Verify the refcnt invariant of cached pages */ if (!(netmem_ref_count(netmem) == 1)) pr_crit("%s() page_pool refcnt %d violation\n", __func__, netmem_ref_count(netmem)); page_pool_return_page(pool, netmem); } } static void __page_pool_destroy(struct page_pool *pool) { if (pool->disconnect) pool->disconnect(pool); page_pool_unlist(pool); page_pool_uninit(pool); if (pool->mp_priv) { mp_dmabuf_devmem_destroy(pool); static_branch_dec(&page_pool_mem_providers); } kfree(pool); } static void page_pool_empty_alloc_cache_once(struct page_pool *pool) { netmem_ref netmem; if (pool->destroy_cnt) return; /* Empty alloc cache, assume caller made sure this is * no-longer in use, and page_pool_alloc_pages() cannot be * call concurrently. */ while (pool->alloc.count) { netmem = pool->alloc.cache[--pool->alloc.count]; page_pool_return_page(pool, netmem); } } static void page_pool_scrub(struct page_pool *pool) { page_pool_empty_alloc_cache_once(pool); pool->destroy_cnt++; /* No more consumers should exist, but producers could still * be in-flight. */ page_pool_empty_ring(pool); } static int page_pool_release(struct page_pool *pool) { int inflight; page_pool_scrub(pool); inflight = page_pool_inflight(pool, true); if (!inflight) __page_pool_destroy(pool); return inflight; } static void page_pool_release_retry(struct work_struct *wq) { struct delayed_work *dwq = to_delayed_work(wq); struct page_pool *pool = container_of(dwq, typeof(*pool), release_dw); void *netdev; int inflight; inflight = page_pool_release(pool); if (!inflight) return; /* Periodic warning for page pools the user can't see */ netdev = READ_ONCE(pool->slow.netdev); if (time_after_eq(jiffies, pool->defer_warn) && (!netdev || netdev == NET_PTR_POISON)) { int sec = (s32)((u32)jiffies - (u32)pool->defer_start) / HZ; pr_warn("%s() stalled pool shutdown: id %u, %d inflight %d sec\n", __func__, pool->user.id, inflight, sec); pool->defer_warn = jiffies + DEFER_WARN_INTERVAL; } /* Still not ready to be disconnected, retry later */ schedule_delayed_work(&pool->release_dw, DEFER_TIME); } void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *), const struct xdp_mem_info *mem) { refcount_inc(&pool->user_cnt); pool->disconnect = disconnect; pool->xdp_mem_id = mem->id; } void page_pool_disable_direct_recycling(struct page_pool *pool) { /* Disable direct recycling based on pool->cpuid. * Paired with READ_ONCE() in page_pool_napi_local(). */ WRITE_ONCE(pool->cpuid, -1); if (!pool->p.napi) return; /* To avoid races with recycling and additional barriers make sure * pool and NAPI are unlinked when NAPI is disabled. */ WARN_ON(!test_bit(NAPI_STATE_SCHED, &pool->p.napi->state)); WARN_ON(READ_ONCE(pool->p.napi->list_owner) != -1); WRITE_ONCE(pool->p.napi, NULL); } EXPORT_SYMBOL(page_pool_disable_direct_recycling); void page_pool_destroy(struct page_pool *pool) { if (!pool) return; if (!page_pool_put(pool)) return; page_pool_disable_direct_recycling(pool); page_pool_free_frag(pool); if (!page_pool_release(pool)) return; page_pool_detached(pool); pool->defer_start = jiffies; pool->defer_warn = jiffies + DEFER_WARN_INTERVAL; INIT_DELAYED_WORK(&pool->release_dw, page_pool_release_retry); schedule_delayed_work(&pool->release_dw, DEFER_TIME); } EXPORT_SYMBOL(page_pool_destroy); /* Caller must provide appropriate safe context, e.g. NAPI. */ void page_pool_update_nid(struct page_pool *pool, int new_nid) { netmem_ref netmem; trace_page_pool_update_nid(pool, new_nid); pool->p.nid = new_nid; /* Flush pool alloc cache, as refill will check NUMA node */ while (pool->alloc.count) { netmem = pool->alloc.cache[--pool->alloc.count]; page_pool_return_page(pool, netmem); } } EXPORT_SYMBOL(page_pool_update_nid); |
| 20 20 2 2 12 1 11 19 12 2 9 9 7 1 2 4 3 1 1 1 1 7 7 1 6 1 5 5 1 1 1 15 1 1 3 1 23 1 10 1 31 1 1 4 1 1 9 9 1 2 1 8 3 1 3 1 1 1 5 1 1 3 1 1 8 6 2 2 1 1 4 2 1 4 1 1 2 1 21 1 1 4 5 6 1 1 1 7 3 4 7 167 1 10 18 5 35 21 12 6 7 5 10 7 1 7 25 11 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 | // SPDX-License-Identifier: GPL-2.0-only /* * VMware VMCI Driver * * Copyright (C) 2012 VMware, Inc. All rights reserved. */ #include <linux/vmw_vmci_defs.h> #include <linux/vmw_vmci_api.h> #include <linux/miscdevice.h> #include <linux/interrupt.h> #include <linux/highmem.h> #include <linux/atomic.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/sched.h> #include <linux/cred.h> #include <linux/slab.h> #include <linux/file.h> #include <linux/init.h> #include <linux/poll.h> #include <linux/pci.h> #include <linux/smp.h> #include <linux/fs.h> #include <linux/io.h> #include "vmci_handle_array.h" #include "vmci_queue_pair.h" #include "vmci_datagram.h" #include "vmci_doorbell.h" #include "vmci_resource.h" #include "vmci_context.h" #include "vmci_driver.h" #include "vmci_event.h" #define VMCI_UTIL_NUM_RESOURCES 1 enum { VMCI_NOTIFY_RESOURCE_QUEUE_PAIR = 0, VMCI_NOTIFY_RESOURCE_DOOR_BELL = 1, }; enum { VMCI_NOTIFY_RESOURCE_ACTION_NOTIFY = 0, VMCI_NOTIFY_RESOURCE_ACTION_CREATE = 1, VMCI_NOTIFY_RESOURCE_ACTION_DESTROY = 2, }; /* * VMCI driver initialization. This block can also be used to * pass initial group membership etc. */ struct vmci_init_blk { u32 cid; u32 flags; }; /* VMCIqueue_pairAllocInfo_VMToVM */ struct vmci_qp_alloc_info_vmvm { struct vmci_handle handle; u32 peer; u32 flags; u64 produce_size; u64 consume_size; u64 produce_page_file; /* User VA. */ u64 consume_page_file; /* User VA. */ u64 produce_page_file_size; /* Size of the file name array. */ u64 consume_page_file_size; /* Size of the file name array. */ s32 result; u32 _pad; }; /* VMCISetNotifyInfo: Used to pass notify flag's address to the host driver. */ struct vmci_set_notify_info { u64 notify_uva; s32 result; u32 _pad; }; /* * Per-instance host state */ struct vmci_host_dev { struct vmci_ctx *context; int user_version; enum vmci_obj_type ct_type; struct mutex lock; /* Mutex lock for vmci context access */ }; static struct vmci_ctx *host_context; static bool vmci_host_device_initialized; static atomic_t vmci_host_active_users = ATOMIC_INIT(0); /* * Determines whether the VMCI host personality is * available. Since the core functionality of the host driver is * always present, all guests could possibly use the host * personality. However, to minimize the deviation from the * pre-unified driver state of affairs, we only consider the host * device active if there is no active guest device or if there * are VMX'en with active VMCI contexts using the host device. */ bool vmci_host_code_active(void) { return vmci_host_device_initialized && (!vmci_guest_code_active() || atomic_read(&vmci_host_active_users) > 0); } int vmci_host_users(void) { return atomic_read(&vmci_host_active_users); } /* * Called on open of /dev/vmci. */ static int vmci_host_open(struct inode *inode, struct file *filp) { struct vmci_host_dev *vmci_host_dev; vmci_host_dev = kzalloc(sizeof(struct vmci_host_dev), GFP_KERNEL); if (vmci_host_dev == NULL) return -ENOMEM; vmci_host_dev->ct_type = VMCIOBJ_NOT_SET; mutex_init(&vmci_host_dev->lock); filp->private_data = vmci_host_dev; return 0; } /* * Called on close of /dev/vmci, most often when the process * exits. */ static int vmci_host_close(struct inode *inode, struct file *filp) { struct vmci_host_dev *vmci_host_dev = filp->private_data; if (vmci_host_dev->ct_type == VMCIOBJ_CONTEXT) { vmci_ctx_destroy(vmci_host_dev->context); vmci_host_dev->context = NULL; /* * The number of active contexts is used to track whether any * VMX'en are using the host personality. It is incremented when * a context is created through the IOCTL_VMCI_INIT_CONTEXT * ioctl. */ atomic_dec(&vmci_host_active_users); } vmci_host_dev->ct_type = VMCIOBJ_NOT_SET; kfree(vmci_host_dev); filp->private_data = NULL; return 0; } /* * This is used to wake up the VMX when a VMCI call arrives, or * to wake up select() or poll() at the next clock tick. */ static __poll_t vmci_host_poll(struct file *filp, poll_table *wait) { struct vmci_host_dev *vmci_host_dev = filp->private_data; struct vmci_ctx *context; __poll_t mask = 0; if (vmci_host_dev->ct_type == VMCIOBJ_CONTEXT) { /* * Read context only if ct_type == VMCIOBJ_CONTEXT to make * sure that context is initialized */ context = vmci_host_dev->context; /* Check for VMCI calls to this VM context. */ if (wait) poll_wait(filp, &context->host_context.wait_queue, wait); spin_lock(&context->lock); if (context->pending_datagrams > 0 || vmci_handle_arr_get_size( context->pending_doorbell_array) > 0) { mask = EPOLLIN; } spin_unlock(&context->lock); } return mask; } /* * Copies the handles of a handle array into a user buffer, and * returns the new length in userBufferSize. If the copy to the * user buffer fails, the functions still returns VMCI_SUCCESS, * but retval != 0. */ static int drv_cp_harray_to_user(void __user *user_buf_uva, u64 *user_buf_size, struct vmci_handle_arr *handle_array, int *retval) { u32 array_size = 0; struct vmci_handle *handles; if (handle_array) array_size = vmci_handle_arr_get_size(handle_array); if (array_size * sizeof(*handles) > *user_buf_size) return VMCI_ERROR_MORE_DATA; *user_buf_size = array_size * sizeof(*handles); if (*user_buf_size) *retval = copy_to_user(user_buf_uva, vmci_handle_arr_get_handles (handle_array), *user_buf_size); return VMCI_SUCCESS; } /* * Sets up a given context for notify to work. Maps the notify * boolean in user VA into kernel space. */ static int vmci_host_setup_notify(struct vmci_ctx *context, unsigned long uva) { int retval; if (context->notify_page) { pr_devel("%s: Notify mechanism is already set up\n", __func__); return VMCI_ERROR_DUPLICATE_ENTRY; } /* * We are using 'bool' internally, but let's make sure we explicit * about the size. */ BUILD_BUG_ON(sizeof(bool) != sizeof(u8)); /* * Lock physical page backing a given user VA. */ retval = get_user_pages_fast(uva, 1, FOLL_WRITE, &context->notify_page); if (retval != 1) { context->notify_page = NULL; return VMCI_ERROR_GENERIC; } if (context->notify_page == NULL) return VMCI_ERROR_UNAVAILABLE; /* * Map the locked page and set up notify pointer. */ context->notify = kmap(context->notify_page) + (uva & (PAGE_SIZE - 1)); vmci_ctx_check_signal_notify(context); return VMCI_SUCCESS; } static int vmci_host_get_version(struct vmci_host_dev *vmci_host_dev, unsigned int cmd, void __user *uptr) { if (cmd == IOCTL_VMCI_VERSION2) { int __user *vptr = uptr; if (get_user(vmci_host_dev->user_version, vptr)) return -EFAULT; } /* * The basic logic here is: * * If the user sends in a version of 0 tell it our version. * If the user didn't send in a version, tell it our version. * If the user sent in an old version, tell it -its- version. * If the user sent in an newer version, tell it our version. * * The rationale behind telling the caller its version is that * Workstation 6.5 required that VMX and VMCI kernel module were * version sync'd. All new VMX users will be programmed to * handle the VMCI kernel module version. */ if (vmci_host_dev->user_version > 0 && vmci_host_dev->user_version < VMCI_VERSION_HOSTQP) { return vmci_host_dev->user_version; } return VMCI_VERSION; } #define vmci_ioctl_err(fmt, ...) \ pr_devel("%s: " fmt, ioctl_name, ##__VA_ARGS__) static int vmci_host_do_init_context(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_init_blk init_block; const struct cred *cred; int retval; if (copy_from_user(&init_block, uptr, sizeof(init_block))) { vmci_ioctl_err("error reading init block\n"); return -EFAULT; } mutex_lock(&vmci_host_dev->lock); if (vmci_host_dev->ct_type != VMCIOBJ_NOT_SET) { vmci_ioctl_err("received VMCI init on initialized handle\n"); retval = -EINVAL; goto out; } if (init_block.flags & ~VMCI_PRIVILEGE_FLAG_RESTRICTED) { vmci_ioctl_err("unsupported VMCI restriction flag\n"); retval = -EINVAL; goto out; } cred = get_current_cred(); vmci_host_dev->context = vmci_ctx_create(init_block.cid, init_block.flags, 0, vmci_host_dev->user_version, cred); put_cred(cred); if (IS_ERR(vmci_host_dev->context)) { retval = PTR_ERR(vmci_host_dev->context); vmci_ioctl_err("error initializing context\n"); goto out; } /* * Copy cid to userlevel, we do this to allow the VMX * to enforce its policy on cid generation. */ init_block.cid = vmci_ctx_get_id(vmci_host_dev->context); if (copy_to_user(uptr, &init_block, sizeof(init_block))) { vmci_ctx_destroy(vmci_host_dev->context); vmci_host_dev->context = NULL; vmci_ioctl_err("error writing init block\n"); retval = -EFAULT; goto out; } vmci_host_dev->ct_type = VMCIOBJ_CONTEXT; atomic_inc(&vmci_host_active_users); vmci_call_vsock_callback(true); retval = 0; out: mutex_unlock(&vmci_host_dev->lock); return retval; } static int vmci_host_do_send_datagram(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_datagram_snd_rcv_info send_info; struct vmci_datagram *dg = NULL; u32 cid; if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (copy_from_user(&send_info, uptr, sizeof(send_info))) return -EFAULT; if (send_info.len > VMCI_MAX_DG_SIZE) { vmci_ioctl_err("datagram is too big (size=%d)\n", send_info.len); return -EINVAL; } if (send_info.len < sizeof(*dg)) { vmci_ioctl_err("datagram is too small (size=%d)\n", send_info.len); return -EINVAL; } dg = memdup_user((void __user *)(uintptr_t)send_info.addr, send_info.len); if (IS_ERR(dg)) { vmci_ioctl_err( "cannot allocate memory to dispatch datagram\n"); return PTR_ERR(dg); } if (VMCI_DG_SIZE(dg) != send_info.len) { vmci_ioctl_err("datagram size mismatch\n"); kfree(dg); return -EINVAL; } pr_devel("Datagram dst (handle=0x%x:0x%x) src (handle=0x%x:0x%x), payload (size=%llu bytes)\n", dg->dst.context, dg->dst.resource, dg->src.context, dg->src.resource, (unsigned long long)dg->payload_size); /* Get source context id. */ cid = vmci_ctx_get_id(vmci_host_dev->context); send_info.result = vmci_datagram_dispatch(cid, dg, true); kfree(dg); return copy_to_user(uptr, &send_info, sizeof(send_info)) ? -EFAULT : 0; } static int vmci_host_do_receive_datagram(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_datagram_snd_rcv_info recv_info; struct vmci_datagram *dg = NULL; int retval; size_t size; if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (copy_from_user(&recv_info, uptr, sizeof(recv_info))) return -EFAULT; size = recv_info.len; recv_info.result = vmci_ctx_dequeue_datagram(vmci_host_dev->context, &size, &dg); if (recv_info.result >= VMCI_SUCCESS) { void __user *ubuf = (void __user *)(uintptr_t)recv_info.addr; retval = copy_to_user(ubuf, dg, VMCI_DG_SIZE(dg)); kfree(dg); if (retval != 0) return -EFAULT; } return copy_to_user(uptr, &recv_info, sizeof(recv_info)) ? -EFAULT : 0; } static int vmci_host_do_alloc_queuepair(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_handle handle; int vmci_status; int __user *retptr; if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (vmci_host_dev->user_version < VMCI_VERSION_NOVMVM) { struct vmci_qp_alloc_info_vmvm alloc_info; struct vmci_qp_alloc_info_vmvm __user *info = uptr; if (copy_from_user(&alloc_info, uptr, sizeof(alloc_info))) return -EFAULT; handle = alloc_info.handle; retptr = &info->result; vmci_status = vmci_qp_broker_alloc(alloc_info.handle, alloc_info.peer, alloc_info.flags, VMCI_NO_PRIVILEGE_FLAGS, alloc_info.produce_size, alloc_info.consume_size, NULL, vmci_host_dev->context); if (vmci_status == VMCI_SUCCESS) vmci_status = VMCI_SUCCESS_QUEUEPAIR_CREATE; } else { struct vmci_qp_alloc_info alloc_info; struct vmci_qp_alloc_info __user *info = uptr; struct vmci_qp_page_store page_store; if (copy_from_user(&alloc_info, uptr, sizeof(alloc_info))) return -EFAULT; handle = alloc_info.handle; retptr = &info->result; page_store.pages = alloc_info.ppn_va; page_store.len = alloc_info.num_ppns; vmci_status = vmci_qp_broker_alloc(alloc_info.handle, alloc_info.peer, alloc_info.flags, VMCI_NO_PRIVILEGE_FLAGS, alloc_info.produce_size, alloc_info.consume_size, &page_store, vmci_host_dev->context); } if (put_user(vmci_status, retptr)) { if (vmci_status >= VMCI_SUCCESS) { vmci_status = vmci_qp_broker_detach(handle, vmci_host_dev->context); } return -EFAULT; } return 0; } static int vmci_host_do_queuepair_setva(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_qp_set_va_info set_va_info; struct vmci_qp_set_va_info __user *info = uptr; s32 result; if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (vmci_host_dev->user_version < VMCI_VERSION_NOVMVM) { vmci_ioctl_err("is not allowed\n"); return -EINVAL; } if (copy_from_user(&set_va_info, uptr, sizeof(set_va_info))) return -EFAULT; if (set_va_info.va) { /* * VMX is passing down a new VA for the queue * pair mapping. */ result = vmci_qp_broker_map(set_va_info.handle, vmci_host_dev->context, set_va_info.va); } else { /* * The queue pair is about to be unmapped by * the VMX. */ result = vmci_qp_broker_unmap(set_va_info.handle, vmci_host_dev->context, 0); } return put_user(result, &info->result) ? -EFAULT : 0; } static int vmci_host_do_queuepair_setpf(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_qp_page_file_info page_file_info; struct vmci_qp_page_file_info __user *info = uptr; s32 result; if (vmci_host_dev->user_version < VMCI_VERSION_HOSTQP || vmci_host_dev->user_version >= VMCI_VERSION_NOVMVM) { vmci_ioctl_err("not supported on this VMX (version=%d)\n", vmci_host_dev->user_version); return -EINVAL; } if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (copy_from_user(&page_file_info, uptr, sizeof(*info))) return -EFAULT; /* * Communicate success pre-emptively to the caller. Note that the * basic premise is that it is incumbent upon the caller not to look at * the info.result field until after the ioctl() returns. And then, * only if the ioctl() result indicates no error. We send up the * SUCCESS status before calling SetPageStore() store because failing * to copy up the result code means unwinding the SetPageStore(). * * It turns out the logic to unwind a SetPageStore() opens a can of * worms. For example, if a host had created the queue_pair and a * guest attaches and SetPageStore() is successful but writing success * fails, then ... the host has to be stopped from writing (anymore) * data into the queue_pair. That means an additional test in the * VMCI_Enqueue() code path. Ugh. */ if (put_user(VMCI_SUCCESS, &info->result)) { /* * In this case, we can't write a result field of the * caller's info block. So, we don't even try to * SetPageStore(). */ return -EFAULT; } result = vmci_qp_broker_set_page_store(page_file_info.handle, page_file_info.produce_va, page_file_info.consume_va, vmci_host_dev->context); if (result < VMCI_SUCCESS) { if (put_user(result, &info->result)) { /* * Note that in this case the SetPageStore() * call failed but we were unable to * communicate that to the caller (because the * copy_to_user() call failed). So, if we * simply return an error (in this case * -EFAULT) then the caller will know that the * SetPageStore failed even though we couldn't * put the result code in the result field and * indicate exactly why it failed. * * That says nothing about the issue where we * were once able to write to the caller's info * memory and now can't. Something more * serious is probably going on than the fact * that SetPageStore() didn't work. */ return -EFAULT; } } return 0; } static int vmci_host_do_qp_detach(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_qp_dtch_info detach_info; struct vmci_qp_dtch_info __user *info = uptr; s32 result; if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (copy_from_user(&detach_info, uptr, sizeof(detach_info))) return -EFAULT; result = vmci_qp_broker_detach(detach_info.handle, vmci_host_dev->context); if (result == VMCI_SUCCESS && vmci_host_dev->user_version < VMCI_VERSION_NOVMVM) { result = VMCI_SUCCESS_LAST_DETACH; } return put_user(result, &info->result) ? -EFAULT : 0; } static int vmci_host_do_ctx_add_notify(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_ctx_info ar_info; struct vmci_ctx_info __user *info = uptr; s32 result; u32 cid; if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (copy_from_user(&ar_info, uptr, sizeof(ar_info))) return -EFAULT; cid = vmci_ctx_get_id(vmci_host_dev->context); result = vmci_ctx_add_notification(cid, ar_info.remote_cid); return put_user(result, &info->result) ? -EFAULT : 0; } static int vmci_host_do_ctx_remove_notify(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_ctx_info ar_info; struct vmci_ctx_info __user *info = uptr; u32 cid; int result; if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (copy_from_user(&ar_info, uptr, sizeof(ar_info))) return -EFAULT; cid = vmci_ctx_get_id(vmci_host_dev->context); result = vmci_ctx_remove_notification(cid, ar_info.remote_cid); return put_user(result, &info->result) ? -EFAULT : 0; } static int vmci_host_do_ctx_get_cpt_state(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_ctx_chkpt_buf_info get_info; u32 cid; void *cpt_buf; int retval; if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (copy_from_user(&get_info, uptr, sizeof(get_info))) return -EFAULT; cid = vmci_ctx_get_id(vmci_host_dev->context); get_info.result = vmci_ctx_get_chkpt_state(cid, get_info.cpt_type, &get_info.buf_size, &cpt_buf); if (get_info.result == VMCI_SUCCESS && get_info.buf_size) { void __user *ubuf = (void __user *)(uintptr_t)get_info.cpt_buf; retval = copy_to_user(ubuf, cpt_buf, get_info.buf_size); kfree(cpt_buf); if (retval) return -EFAULT; } return copy_to_user(uptr, &get_info, sizeof(get_info)) ? -EFAULT : 0; } static int vmci_host_do_ctx_set_cpt_state(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_ctx_chkpt_buf_info set_info; u32 cid; void *cpt_buf; int retval; if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (copy_from_user(&set_info, uptr, sizeof(set_info))) return -EFAULT; cpt_buf = memdup_user((void __user *)(uintptr_t)set_info.cpt_buf, set_info.buf_size); if (IS_ERR(cpt_buf)) return PTR_ERR(cpt_buf); cid = vmci_ctx_get_id(vmci_host_dev->context); set_info.result = vmci_ctx_set_chkpt_state(cid, set_info.cpt_type, set_info.buf_size, cpt_buf); retval = copy_to_user(uptr, &set_info, sizeof(set_info)) ? -EFAULT : 0; kfree(cpt_buf); return retval; } static int vmci_host_do_get_context_id(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { u32 __user *u32ptr = uptr; return put_user(VMCI_HOST_CONTEXT_ID, u32ptr) ? -EFAULT : 0; } static int vmci_host_do_set_notify(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_set_notify_info notify_info; if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (copy_from_user(¬ify_info, uptr, sizeof(notify_info))) return -EFAULT; if (notify_info.notify_uva) { notify_info.result = vmci_host_setup_notify(vmci_host_dev->context, notify_info.notify_uva); } else { vmci_ctx_unset_notify(vmci_host_dev->context); notify_info.result = VMCI_SUCCESS; } return copy_to_user(uptr, ¬ify_info, sizeof(notify_info)) ? -EFAULT : 0; } static int vmci_host_do_notify_resource(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_dbell_notify_resource_info info; u32 cid; if (vmci_host_dev->user_version < VMCI_VERSION_NOTIFY) { vmci_ioctl_err("invalid for current VMX versions\n"); return -EINVAL; } if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (copy_from_user(&info, uptr, sizeof(info))) return -EFAULT; cid = vmci_ctx_get_id(vmci_host_dev->context); switch (info.action) { case VMCI_NOTIFY_RESOURCE_ACTION_NOTIFY: if (info.resource == VMCI_NOTIFY_RESOURCE_DOOR_BELL) { u32 flags = VMCI_NO_PRIVILEGE_FLAGS; info.result = vmci_ctx_notify_dbell(cid, info.handle, flags); } else { info.result = VMCI_ERROR_UNAVAILABLE; } break; case VMCI_NOTIFY_RESOURCE_ACTION_CREATE: info.result = vmci_ctx_dbell_create(cid, info.handle); break; case VMCI_NOTIFY_RESOURCE_ACTION_DESTROY: info.result = vmci_ctx_dbell_destroy(cid, info.handle); break; default: vmci_ioctl_err("got unknown action (action=%d)\n", info.action); info.result = VMCI_ERROR_INVALID_ARGS; } return copy_to_user(uptr, &info, sizeof(info)) ? -EFAULT : 0; } static int vmci_host_do_recv_notifications(struct vmci_host_dev *vmci_host_dev, const char *ioctl_name, void __user *uptr) { struct vmci_ctx_notify_recv_info info; struct vmci_handle_arr *db_handle_array; struct vmci_handle_arr *qp_handle_array; void __user *ubuf; u32 cid; int retval = 0; if (vmci_host_dev->ct_type != VMCIOBJ_CONTEXT) { vmci_ioctl_err("only valid for contexts\n"); return -EINVAL; } if (vmci_host_dev->user_version < VMCI_VERSION_NOTIFY) { vmci_ioctl_err("not supported for the current vmx version\n"); return -EINVAL; } if (copy_from_user(&info, uptr, sizeof(info))) return -EFAULT; if ((info.db_handle_buf_size && !info.db_handle_buf_uva) || (info.qp_handle_buf_size && !info.qp_handle_buf_uva)) { return -EINVAL; } cid = vmci_ctx_get_id(vmci_host_dev->context); info.result = vmci_ctx_rcv_notifications_get(cid, &db_handle_array, &qp_handle_array); if (info.result != VMCI_SUCCESS) return copy_to_user(uptr, &info, sizeof(info)) ? -EFAULT : 0; ubuf = (void __user *)(uintptr_t)info.db_handle_buf_uva; info.result = drv_cp_harray_to_user(ubuf, &info.db_handle_buf_size, db_handle_array, &retval); if (info.result == VMCI_SUCCESS && !retval) { ubuf = (void __user *)(uintptr_t)info.qp_handle_buf_uva; info.result = drv_cp_harray_to_user(ubuf, &info.qp_handle_buf_size, qp_handle_array, &retval); } if (!retval && copy_to_user(uptr, &info, sizeof(info))) retval = -EFAULT; vmci_ctx_rcv_notifications_release(cid, db_handle_array, qp_handle_array, info.result == VMCI_SUCCESS && !retval); return retval; } static long vmci_host_unlocked_ioctl(struct file *filp, unsigned int iocmd, unsigned long ioarg) { #define VMCI_DO_IOCTL(ioctl_name, ioctl_fn) do { \ char *name = "IOCTL_VMCI_" # ioctl_name; \ return vmci_host_do_ ## ioctl_fn( \ vmci_host_dev, name, uptr); \ } while (0) struct vmci_host_dev *vmci_host_dev = filp->private_data; void __user *uptr = (void __user *)ioarg; switch (iocmd) { case IOCTL_VMCI_INIT_CONTEXT: VMCI_DO_IOCTL(INIT_CONTEXT, init_context); case IOCTL_VMCI_DATAGRAM_SEND: VMCI_DO_IOCTL(DATAGRAM_SEND, send_datagram); case IOCTL_VMCI_DATAGRAM_RECEIVE: VMCI_DO_IOCTL(DATAGRAM_RECEIVE, receive_datagram); case IOCTL_VMCI_QUEUEPAIR_ALLOC: VMCI_DO_IOCTL(QUEUEPAIR_ALLOC, alloc_queuepair); case IOCTL_VMCI_QUEUEPAIR_SETVA: VMCI_DO_IOCTL(QUEUEPAIR_SETVA, queuepair_setva); case IOCTL_VMCI_QUEUEPAIR_SETPAGEFILE: VMCI_DO_IOCTL(QUEUEPAIR_SETPAGEFILE, queuepair_setpf); case IOCTL_VMCI_QUEUEPAIR_DETACH: VMCI_DO_IOCTL(QUEUEPAIR_DETACH, qp_detach); case IOCTL_VMCI_CTX_ADD_NOTIFICATION: VMCI_DO_IOCTL(CTX_ADD_NOTIFICATION, ctx_add_notify); case IOCTL_VMCI_CTX_REMOVE_NOTIFICATION: VMCI_DO_IOCTL(CTX_REMOVE_NOTIFICATION, ctx_remove_notify); case IOCTL_VMCI_CTX_GET_CPT_STATE: VMCI_DO_IOCTL(CTX_GET_CPT_STATE, ctx_get_cpt_state); case IOCTL_VMCI_CTX_SET_CPT_STATE: VMCI_DO_IOCTL(CTX_SET_CPT_STATE, ctx_set_cpt_state); case IOCTL_VMCI_GET_CONTEXT_ID: VMCI_DO_IOCTL(GET_CONTEXT_ID, get_context_id); case IOCTL_VMCI_SET_NOTIFY: VMCI_DO_IOCTL(SET_NOTIFY, set_notify); case IOCTL_VMCI_NOTIFY_RESOURCE: VMCI_DO_IOCTL(NOTIFY_RESOURCE, notify_resource); case IOCTL_VMCI_NOTIFICATIONS_RECEIVE: VMCI_DO_IOCTL(NOTIFICATIONS_RECEIVE, recv_notifications); case IOCTL_VMCI_VERSION: case IOCTL_VMCI_VERSION2: return vmci_host_get_version(vmci_host_dev, iocmd, uptr); default: pr_devel("%s: Unknown ioctl (iocmd=%d)\n", __func__, iocmd); return -EINVAL; } #undef VMCI_DO_IOCTL } static const struct file_operations vmuser_fops = { .owner = THIS_MODULE, .open = vmci_host_open, .release = vmci_host_close, .poll = vmci_host_poll, .unlocked_ioctl = vmci_host_unlocked_ioctl, .compat_ioctl = compat_ptr_ioctl, }; static struct miscdevice vmci_host_miscdev = { .name = "vmci", .minor = MISC_DYNAMIC_MINOR, .fops = &vmuser_fops, }; int __init vmci_host_init(void) { int error; host_context = vmci_ctx_create(VMCI_HOST_CONTEXT_ID, VMCI_DEFAULT_PROC_PRIVILEGE_FLAGS, -1, VMCI_VERSION, NULL); if (IS_ERR(host_context)) { error = PTR_ERR(host_context); pr_warn("Failed to initialize VMCIContext (error%d)\n", error); return error; } error = misc_register(&vmci_host_miscdev); if (error) { pr_warn("Module registration error (name=%s, major=%d, minor=%d, err=%d)\n", vmci_host_miscdev.name, MISC_MAJOR, vmci_host_miscdev.minor, error); pr_warn("Unable to initialize host personality\n"); vmci_ctx_destroy(host_context); return error; } pr_info("VMCI host device registered (name=%s, major=%d, minor=%d)\n", vmci_host_miscdev.name, MISC_MAJOR, vmci_host_miscdev.minor); vmci_host_device_initialized = true; return 0; } void __exit vmci_host_exit(void) { vmci_host_device_initialized = false; misc_deregister(&vmci_host_miscdev); vmci_ctx_destroy(host_context); vmci_qp_broker_exit(); pr_debug("VMCI host driver module unloaded\n"); } |
| 8 8 8 13 13 25 2 1 2 19 1 1 2 2 2 1 2 9 5 13 1 13 2 6 5 1 6 6 6 1727 1726 1728 406 129 1 9 9 13 5 9 12 13 13 2 6 2 3 2 6 2 3 52 52 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 | // SPDX-License-Identifier: GPL-2.0-or-later /* * net/sched/act_mirred.c packet mirroring and redirect actions * * Authors: Jamal Hadi Salim (2002-4) * * TODO: Add ingress support (and socket redirect support) */ #include <linux/types.h> #include <linux/kernel.h> #include <linux/string.h> #include <linux/errno.h> #include <linux/skbuff.h> #include <linux/rtnetlink.h> #include <linux/module.h> #include <linux/init.h> #include <linux/gfp.h> #include <linux/if_arp.h> #include <net/net_namespace.h> #include <net/netlink.h> #include <net/dst.h> #include <net/pkt_sched.h> #include <net/pkt_cls.h> #include <linux/tc_act/tc_mirred.h> #include <net/tc_act/tc_mirred.h> #include <net/tc_wrapper.h> static LIST_HEAD(mirred_list); static DEFINE_SPINLOCK(mirred_list_lock); #define MIRRED_NEST_LIMIT 4 static DEFINE_PER_CPU(unsigned int, mirred_nest_level); static bool tcf_mirred_is_act_redirect(int action) { return action == TCA_EGRESS_REDIR || action == TCA_INGRESS_REDIR; } static bool tcf_mirred_act_wants_ingress(int action) { switch (action) { case TCA_EGRESS_REDIR: case TCA_EGRESS_MIRROR: return false; case TCA_INGRESS_REDIR: case TCA_INGRESS_MIRROR: return true; default: BUG(); } } static bool tcf_mirred_can_reinsert(int action) { switch (action) { case TC_ACT_SHOT: case TC_ACT_STOLEN: case TC_ACT_QUEUED: case TC_ACT_TRAP: return true; } return false; } static struct net_device *tcf_mirred_dev_dereference(struct tcf_mirred *m) { return rcu_dereference_protected(m->tcfm_dev, lockdep_is_held(&m->tcf_lock)); } static void tcf_mirred_release(struct tc_action *a) { struct tcf_mirred *m = to_mirred(a); struct net_device *dev; spin_lock(&mirred_list_lock); list_del(&m->tcfm_list); spin_unlock(&mirred_list_lock); /* last reference to action, no need to lock */ dev = rcu_dereference_protected(m->tcfm_dev, 1); netdev_put(dev, &m->tcfm_dev_tracker); } static const struct nla_policy mirred_policy[TCA_MIRRED_MAX + 1] = { [TCA_MIRRED_PARMS] = { .len = sizeof(struct tc_mirred) }, [TCA_MIRRED_BLOCKID] = NLA_POLICY_MIN(NLA_U32, 1), }; static struct tc_action_ops act_mirred_ops; static void tcf_mirred_replace_dev(struct tcf_mirred *m, struct net_device *ndev) { struct net_device *odev; odev = rcu_replace_pointer(m->tcfm_dev, ndev, lockdep_is_held(&m->tcf_lock)); netdev_put(odev, &m->tcfm_dev_tracker); } static int tcf_mirred_init(struct net *net, struct nlattr *nla, struct nlattr *est, struct tc_action **a, struct tcf_proto *tp, u32 flags, struct netlink_ext_ack *extack) { struct tc_action_net *tn = net_generic(net, act_mirred_ops.net_id); bool bind = flags & TCA_ACT_FLAGS_BIND; struct nlattr *tb[TCA_MIRRED_MAX + 1]; struct tcf_chain *goto_ch = NULL; bool mac_header_xmit = false; struct tc_mirred *parm; struct tcf_mirred *m; bool exists = false; int ret, err; u32 index; if (!nla) { NL_SET_ERR_MSG_MOD(extack, "Mirred requires attributes to be passed"); return -EINVAL; } ret = nla_parse_nested_deprecated(tb, TCA_MIRRED_MAX, nla, mirred_policy, extack); if (ret < 0) return ret; if (!tb[TCA_MIRRED_PARMS]) { NL_SET_ERR_MSG_MOD(extack, "Missing required mirred parameters"); return -EINVAL; } parm = nla_data(tb[TCA_MIRRED_PARMS]); index = parm->index; err = tcf_idr_check_alloc(tn, &index, a, bind); if (err < 0) return err; exists = err; if (exists && bind) return ACT_P_BOUND; if (tb[TCA_MIRRED_BLOCKID] && parm->ifindex) { NL_SET_ERR_MSG_MOD(extack, "Cannot specify Block ID and dev simultaneously"); if (exists) tcf_idr_release(*a, bind); else tcf_idr_cleanup(tn, index); return -EINVAL; } switch (parm->eaction) { case TCA_EGRESS_MIRROR: case TCA_EGRESS_REDIR: case TCA_INGRESS_REDIR: case TCA_INGRESS_MIRROR: break; default: if (exists) tcf_idr_release(*a, bind); else tcf_idr_cleanup(tn, index); NL_SET_ERR_MSG_MOD(extack, "Unknown mirred option"); return -EINVAL; } if (!exists) { if (!parm->ifindex && !tb[TCA_MIRRED_BLOCKID]) { tcf_idr_cleanup(tn, index); NL_SET_ERR_MSG_MOD(extack, "Must specify device or block"); return -EINVAL; } ret = tcf_idr_create_from_flags(tn, index, est, a, &act_mirred_ops, bind, flags); if (ret) { tcf_idr_cleanup(tn, index); return ret; } ret = ACT_P_CREATED; } else if (!(flags & TCA_ACT_FLAGS_REPLACE)) { tcf_idr_release(*a, bind); return -EEXIST; } m = to_mirred(*a); if (ret == ACT_P_CREATED) INIT_LIST_HEAD(&m->tcfm_list); err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); if (err < 0) goto release_idr; spin_lock_bh(&m->tcf_lock); if (parm->ifindex) { struct net_device *ndev; ndev = dev_get_by_index(net, parm->ifindex); if (!ndev) { spin_unlock_bh(&m->tcf_lock); err = -ENODEV; goto put_chain; } mac_header_xmit = dev_is_mac_header_xmit(ndev); tcf_mirred_replace_dev(m, ndev); netdev_tracker_alloc(ndev, &m->tcfm_dev_tracker, GFP_ATOMIC); m->tcfm_mac_header_xmit = mac_header_xmit; m->tcfm_blockid = 0; } else if (tb[TCA_MIRRED_BLOCKID]) { tcf_mirred_replace_dev(m, NULL); m->tcfm_mac_header_xmit = false; m->tcfm_blockid = nla_get_u32(tb[TCA_MIRRED_BLOCKID]); } goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); m->tcfm_eaction = parm->eaction; spin_unlock_bh(&m->tcf_lock); if (goto_ch) tcf_chain_put_by_act(goto_ch); if (ret == ACT_P_CREATED) { spin_lock(&mirred_list_lock); list_add(&m->tcfm_list, &mirred_list); spin_unlock(&mirred_list_lock); } return ret; put_chain: if (goto_ch) tcf_chain_put_by_act(goto_ch); release_idr: tcf_idr_release(*a, bind); return err; } static int tcf_mirred_forward(bool at_ingress, bool want_ingress, struct sk_buff *skb) { int err; if (!want_ingress) err = tcf_dev_queue_xmit(skb, dev_queue_xmit); else if (!at_ingress) err = netif_rx(skb); else err = netif_receive_skb(skb); return err; } static int tcf_mirred_to_dev(struct sk_buff *skb, struct tcf_mirred *m, struct net_device *dev, const bool m_mac_header_xmit, int m_eaction, int retval) { struct sk_buff *skb_to_send = skb; bool want_ingress; bool is_redirect; bool expects_nh; bool at_ingress; bool dont_clone; int mac_len; bool at_nh; int err; is_redirect = tcf_mirred_is_act_redirect(m_eaction); if (unlikely(!(dev->flags & IFF_UP)) || !netif_carrier_ok(dev)) { net_notice_ratelimited("tc mirred to Houston: device %s is down\n", dev->name); goto err_cant_do; } /* we could easily avoid the clone only if called by ingress and clsact; * since we can't easily detect the clsact caller, skip clone only for * ingress - that covers the TC S/W datapath. */ at_ingress = skb_at_tc_ingress(skb); dont_clone = skb_at_tc_ingress(skb) && is_redirect && tcf_mirred_can_reinsert(retval); if (!dont_clone) { skb_to_send = skb_clone(skb, GFP_ATOMIC); if (!skb_to_send) goto err_cant_do; } want_ingress = tcf_mirred_act_wants_ingress(m_eaction); /* All mirred/redirected skbs should clear previous ct info */ nf_reset_ct(skb_to_send); if (want_ingress && !at_ingress) /* drop dst for egress -> ingress */ skb_dst_drop(skb_to_send); expects_nh = want_ingress || !m_mac_header_xmit; at_nh = skb->data == skb_network_header(skb); if (at_nh != expects_nh) { mac_len = at_ingress ? skb->mac_len : skb_network_offset(skb); if (expects_nh) { /* target device/action expect data at nh */ skb_pull_rcsum(skb_to_send, mac_len); } else { /* target device/action expect data at mac */ skb_push_rcsum(skb_to_send, mac_len); } } skb_to_send->skb_iif = skb->dev->ifindex; skb_to_send->dev = dev; if (is_redirect) { if (skb == skb_to_send) retval = TC_ACT_CONSUMED; skb_set_redirected(skb_to_send, skb_to_send->tc_at_ingress); err = tcf_mirred_forward(at_ingress, want_ingress, skb_to_send); } else { err = tcf_mirred_forward(at_ingress, want_ingress, skb_to_send); } if (err) tcf_action_inc_overlimit_qstats(&m->common); return retval; err_cant_do: if (is_redirect) retval = TC_ACT_SHOT; tcf_action_inc_overlimit_qstats(&m->common); return retval; } static int tcf_blockcast_redir(struct sk_buff *skb, struct tcf_mirred *m, struct tcf_block *block, int m_eaction, const u32 exception_ifindex, int retval) { struct net_device *dev_prev = NULL; struct net_device *dev = NULL; unsigned long index; int mirred_eaction; mirred_eaction = tcf_mirred_act_wants_ingress(m_eaction) ? TCA_INGRESS_MIRROR : TCA_EGRESS_MIRROR; xa_for_each(&block->ports, index, dev) { if (index == exception_ifindex) continue; if (!dev_prev) goto assign_prev; tcf_mirred_to_dev(skb, m, dev_prev, dev_is_mac_header_xmit(dev), mirred_eaction, retval); assign_prev: dev_prev = dev; } if (dev_prev) return tcf_mirred_to_dev(skb, m, dev_prev, dev_is_mac_header_xmit(dev_prev), m_eaction, retval); return retval; } static int tcf_blockcast_mirror(struct sk_buff *skb, struct tcf_mirred *m, struct tcf_block *block, int m_eaction, const u32 exception_ifindex, int retval) { struct net_device *dev = NULL; unsigned long index; xa_for_each(&block->ports, index, dev) { if (index == exception_ifindex) continue; tcf_mirred_to_dev(skb, m, dev, dev_is_mac_header_xmit(dev), m_eaction, retval); } return retval; } static int tcf_blockcast(struct sk_buff *skb, struct tcf_mirred *m, const u32 blockid, struct tcf_result *res, int retval) { const u32 exception_ifindex = skb->dev->ifindex; struct tcf_block *block; bool is_redirect; int m_eaction; m_eaction = READ_ONCE(m->tcfm_eaction); is_redirect = tcf_mirred_is_act_redirect(m_eaction); /* we are already under rcu protection, so can call block lookup * directly. */ block = tcf_block_lookup(dev_net(skb->dev), blockid); if (!block || xa_empty(&block->ports)) { tcf_action_inc_overlimit_qstats(&m->common); return retval; } if (is_redirect) return tcf_blockcast_redir(skb, m, block, m_eaction, exception_ifindex, retval); /* If it's not redirect, it is mirror */ return tcf_blockcast_mirror(skb, m, block, m_eaction, exception_ifindex, retval); } TC_INDIRECT_SCOPE int tcf_mirred_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res) { struct tcf_mirred *m = to_mirred(a); int retval = READ_ONCE(m->tcf_action); unsigned int nest_level; bool m_mac_header_xmit; struct net_device *dev; int m_eaction; u32 blockid; nest_level = __this_cpu_inc_return(mirred_nest_level); if (unlikely(nest_level > MIRRED_NEST_LIMIT)) { net_warn_ratelimited("Packet exceeded mirred recursion limit on dev %s\n", netdev_name(skb->dev)); retval = TC_ACT_SHOT; goto dec_nest_level; } tcf_lastuse_update(&m->tcf_tm); tcf_action_update_bstats(&m->common, skb); blockid = READ_ONCE(m->tcfm_blockid); if (blockid) { retval = tcf_blockcast(skb, m, blockid, res, retval); goto dec_nest_level; } dev = rcu_dereference_bh(m->tcfm_dev); if (unlikely(!dev)) { pr_notice_once("tc mirred: target device is gone\n"); tcf_action_inc_overlimit_qstats(&m->common); goto dec_nest_level; } m_mac_header_xmit = READ_ONCE(m->tcfm_mac_header_xmit); m_eaction = READ_ONCE(m->tcfm_eaction); retval = tcf_mirred_to_dev(skb, m, dev, m_mac_header_xmit, m_eaction, retval); dec_nest_level: __this_cpu_dec(mirred_nest_level); return retval; } static void tcf_stats_update(struct tc_action *a, u64 bytes, u64 packets, u64 drops, u64 lastuse, bool hw) { struct tcf_mirred *m = to_mirred(a); struct tcf_t *tm = &m->tcf_tm; tcf_action_update_stats(a, bytes, packets, drops, hw); tm->lastuse = max_t(u64, tm->lastuse, lastuse); } static int tcf_mirred_dump(struct sk_buff *skb, struct tc_action *a, int bind, int ref) { unsigned char *b = skb_tail_pointer(skb); struct tcf_mirred *m = to_mirred(a); struct tc_mirred opt = { .index = m->tcf_index, .refcnt = refcount_read(&m->tcf_refcnt) - ref, .bindcnt = atomic_read(&m->tcf_bindcnt) - bind, }; struct net_device *dev; struct tcf_t t; u32 blockid; spin_lock_bh(&m->tcf_lock); opt.action = m->tcf_action; opt.eaction = m->tcfm_eaction; dev = tcf_mirred_dev_dereference(m); if (dev) opt.ifindex = dev->ifindex; if (nla_put(skb, TCA_MIRRED_PARMS, sizeof(opt), &opt)) goto nla_put_failure; blockid = m->tcfm_blockid; if (blockid && nla_put_u32(skb, TCA_MIRRED_BLOCKID, blockid)) goto nla_put_failure; tcf_tm_dump(&t, &m->tcf_tm); if (nla_put_64bit(skb, TCA_MIRRED_TM, sizeof(t), &t, TCA_MIRRED_PAD)) goto nla_put_failure; spin_unlock_bh(&m->tcf_lock); return skb->len; nla_put_failure: spin_unlock_bh(&m->tcf_lock); nlmsg_trim(skb, b); return -1; } static int mirred_device_event(struct notifier_block *unused, unsigned long event, void *ptr) { struct net_device *dev = netdev_notifier_info_to_dev(ptr); struct tcf_mirred *m; ASSERT_RTNL(); if (event == NETDEV_UNREGISTER) { spin_lock(&mirred_list_lock); list_for_each_entry(m, &mirred_list, tcfm_list) { spin_lock_bh(&m->tcf_lock); if (tcf_mirred_dev_dereference(m) == dev) { netdev_put(dev, &m->tcfm_dev_tracker); /* Note : no rcu grace period necessary, as * net_device are already rcu protected. */ RCU_INIT_POINTER(m->tcfm_dev, NULL); } spin_unlock_bh(&m->tcf_lock); } spin_unlock(&mirred_list_lock); } return NOTIFY_DONE; } static struct notifier_block mirred_device_notifier = { .notifier_call = mirred_device_event, }; static void tcf_mirred_dev_put(void *priv) { struct net_device *dev = priv; dev_put(dev); } static struct net_device * tcf_mirred_get_dev(const struct tc_action *a, tc_action_priv_destructor *destructor) { struct tcf_mirred *m = to_mirred(a); struct net_device *dev; rcu_read_lock(); dev = rcu_dereference(m->tcfm_dev); if (dev) { dev_hold(dev); *destructor = tcf_mirred_dev_put; } rcu_read_unlock(); return dev; } static size_t tcf_mirred_get_fill_size(const struct tc_action *act) { return nla_total_size(sizeof(struct tc_mirred)); } static void tcf_offload_mirred_get_dev(struct flow_action_entry *entry, const struct tc_action *act) { entry->dev = act->ops->get_dev(act, &entry->destructor); if (!entry->dev) return; entry->destructor_priv = entry->dev; } static int tcf_mirred_offload_act_setup(struct tc_action *act, void *entry_data, u32 *index_inc, bool bind, struct netlink_ext_ack *extack) { if (bind) { struct flow_action_entry *entry = entry_data; if (is_tcf_mirred_egress_redirect(act)) { entry->id = FLOW_ACTION_REDIRECT; tcf_offload_mirred_get_dev(entry, act); } else if (is_tcf_mirred_egress_mirror(act)) { entry->id = FLOW_ACTION_MIRRED; tcf_offload_mirred_get_dev(entry, act); } else if (is_tcf_mirred_ingress_redirect(act)) { entry->id = FLOW_ACTION_REDIRECT_INGRESS; tcf_offload_mirred_get_dev(entry, act); } else if (is_tcf_mirred_ingress_mirror(act)) { entry->id = FLOW_ACTION_MIRRED_INGRESS; tcf_offload_mirred_get_dev(entry, act); } else { NL_SET_ERR_MSG_MOD(extack, "Unsupported mirred offload"); return -EOPNOTSUPP; } *index_inc = 1; } else { struct flow_offload_action *fl_action = entry_data; if (is_tcf_mirred_egress_redirect(act)) fl_action->id = FLOW_ACTION_REDIRECT; else if (is_tcf_mirred_egress_mirror(act)) fl_action->id = FLOW_ACTION_MIRRED; else if (is_tcf_mirred_ingress_redirect(act)) fl_action->id = FLOW_ACTION_REDIRECT_INGRESS; else if (is_tcf_mirred_ingress_mirror(act)) fl_action->id = FLOW_ACTION_MIRRED_INGRESS; else return -EOPNOTSUPP; } return 0; } static struct tc_action_ops act_mirred_ops = { .kind = "mirred", .id = TCA_ID_MIRRED, .owner = THIS_MODULE, .act = tcf_mirred_act, .stats_update = tcf_stats_update, .dump = tcf_mirred_dump, .cleanup = tcf_mirred_release, .init = tcf_mirred_init, .get_fill_size = tcf_mirred_get_fill_size, .offload_act_setup = tcf_mirred_offload_act_setup, .size = sizeof(struct tcf_mirred), .get_dev = tcf_mirred_get_dev, }; MODULE_ALIAS_NET_ACT("mirred"); static __net_init int mirred_init_net(struct net *net) { struct tc_action_net *tn = net_generic(net, act_mirred_ops.net_id); return tc_action_net_init(net, tn, &act_mirred_ops); } static void __net_exit mirred_exit_net(struct list_head *net_list) { tc_action_net_exit(net_list, act_mirred_ops.net_id); } static struct pernet_operations mirred_net_ops = { .init = mirred_init_net, .exit_batch = mirred_exit_net, .id = &act_mirred_ops.net_id, .size = sizeof(struct tc_action_net), }; MODULE_AUTHOR("Jamal Hadi Salim(2002)"); MODULE_DESCRIPTION("Device Mirror/redirect actions"); MODULE_LICENSE("GPL"); static int __init mirred_init_module(void) { int err = register_netdevice_notifier(&mirred_device_notifier); if (err) return err; pr_info("Mirror/redirect action on\n"); err = tcf_register_action(&act_mirred_ops, &mirred_net_ops); if (err) unregister_netdevice_notifier(&mirred_device_notifier); return err; } static void __exit mirred_cleanup_module(void) { tcf_unregister_action(&act_mirred_ops, &mirred_net_ops); unregister_netdevice_notifier(&mirred_device_notifier); } module_init(mirred_init_module); module_exit(mirred_cleanup_module); |
| 6 131 131 131 66 55 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 | /* * include/linux/ktime.h * * ktime_t - nanosecond-resolution time format. * * Copyright(C) 2005, Thomas Gleixner <tglx@linutronix.de> * Copyright(C) 2005, Red Hat, Inc., Ingo Molnar * * data type definitions, declarations, prototypes and macros. * * Started by: Thomas Gleixner and Ingo Molnar * * Credits: * * Roman Zippel provided the ideas and primary code snippets of * the ktime_t union and further simplifications of the original * code. * * For licencing details see kernel-base/COPYING */ #ifndef _LINUX_KTIME_H #define _LINUX_KTIME_H #include <asm/bug.h> #include <linux/jiffies.h> #include <linux/time.h> #include <linux/types.h> /** * ktime_set - Set a ktime_t variable from a seconds/nanoseconds value * @secs: seconds to set * @nsecs: nanoseconds to set * * Return: The ktime_t representation of the value. */ static inline ktime_t ktime_set(const s64 secs, const unsigned long nsecs) { if (unlikely(secs >= KTIME_SEC_MAX)) return KTIME_MAX; return secs * NSEC_PER_SEC + (s64)nsecs; } /* Subtract two ktime_t variables. rem = lhs -rhs: */ #define ktime_sub(lhs, rhs) ((lhs) - (rhs)) /* Add two ktime_t variables. res = lhs + rhs: */ #define ktime_add(lhs, rhs) ((lhs) + (rhs)) /* * Same as ktime_add(), but avoids undefined behaviour on overflow; however, * this means that you must check the result for overflow yourself. */ #define ktime_add_unsafe(lhs, rhs) ((u64) (lhs) + (rhs)) /* * Add a ktime_t variable and a scalar nanosecond value. * res = kt + nsval: */ #define ktime_add_ns(kt, nsval) ((kt) + (nsval)) /* * Subtract a scalar nanosecod from a ktime_t variable * res = kt - nsval: */ #define ktime_sub_ns(kt, nsval) ((kt) - (nsval)) /* convert a timespec64 to ktime_t format: */ static inline ktime_t timespec64_to_ktime(struct timespec64 ts) { return ktime_set(ts.tv_sec, ts.tv_nsec); } /* Map the ktime_t to timespec conversion to ns_to_timespec function */ #define ktime_to_timespec64(kt) ns_to_timespec64((kt)) /* Convert ktime_t to nanoseconds */ static inline s64 ktime_to_ns(const ktime_t kt) { return kt; } /** * ktime_compare - Compares two ktime_t variables for less, greater or equal * @cmp1: comparable1 * @cmp2: comparable2 * * Return: ... * cmp1 < cmp2: return <0 * cmp1 == cmp2: return 0 * cmp1 > cmp2: return >0 */ static inline int ktime_compare(const ktime_t cmp1, const ktime_t cmp2) { if (cmp1 < cmp2) return -1; if (cmp1 > cmp2) return 1; return 0; } /** * ktime_after - Compare if a ktime_t value is bigger than another one. * @cmp1: comparable1 * @cmp2: comparable2 * * Return: true if cmp1 happened after cmp2. */ static inline bool ktime_after(const ktime_t cmp1, const ktime_t cmp2) { return ktime_compare(cmp1, cmp2) > 0; } /** * ktime_before - Compare if a ktime_t value is smaller than another one. * @cmp1: comparable1 * @cmp2: comparable2 * * Return: true if cmp1 happened before cmp2. */ static inline bool ktime_before(const ktime_t cmp1, const ktime_t cmp2) { return ktime_compare(cmp1, cmp2) < 0; } #if BITS_PER_LONG < 64 extern s64 __ktime_divns(const ktime_t kt, s64 div); static inline s64 ktime_divns(const ktime_t kt, s64 div) { /* * Negative divisors could cause an inf loop, * so bug out here. */ BUG_ON(div < 0); if (__builtin_constant_p(div) && !(div >> 32)) { s64 ns = kt; u64 tmp = ns < 0 ? -ns : ns; do_div(tmp, div); return ns < 0 ? -tmp : tmp; } else { return __ktime_divns(kt, div); } } #else /* BITS_PER_LONG < 64 */ static inline s64 ktime_divns(const ktime_t kt, s64 div) { /* * 32-bit implementation cannot handle negative divisors, * so catch them on 64bit as well. */ WARN_ON(div < 0); return kt / div; } #endif static inline s64 ktime_to_us(const ktime_t kt) { return ktime_divns(kt, NSEC_PER_USEC); } static inline s64 ktime_to_ms(const ktime_t kt) { return ktime_divns(kt, NSEC_PER_MSEC); } static inline s64 ktime_us_delta(const ktime_t later, const ktime_t earlier) { return ktime_to_us(ktime_sub(later, earlier)); } static inline s64 ktime_ms_delta(const ktime_t later, const ktime_t earlier) { return ktime_to_ms(ktime_sub(later, earlier)); } static inline ktime_t ktime_add_us(const ktime_t kt, const u64 usec) { return ktime_add_ns(kt, usec * NSEC_PER_USEC); } static inline ktime_t ktime_add_ms(const ktime_t kt, const u64 msec) { return ktime_add_ns(kt, msec * NSEC_PER_MSEC); } static inline ktime_t ktime_sub_us(const ktime_t kt, const u64 usec) { return ktime_sub_ns(kt, usec * NSEC_PER_USEC); } static inline ktime_t ktime_sub_ms(const ktime_t kt, const u64 msec) { return ktime_sub_ns(kt, msec * NSEC_PER_MSEC); } extern ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs); /** * ktime_to_timespec64_cond - convert a ktime_t variable to timespec64 * format only if the variable contains data * @kt: the ktime_t variable to convert * @ts: the timespec variable to store the result in * * Return: %true if there was a successful conversion, %false if kt was 0. */ static inline __must_check bool ktime_to_timespec64_cond(const ktime_t kt, struct timespec64 *ts) { if (kt) { *ts = ktime_to_timespec64(kt); return true; } else { return false; } } #include <vdso/ktime.h> static inline ktime_t ns_to_ktime(u64 ns) { return ns; } static inline ktime_t ms_to_ktime(u64 ms) { return ms * NSEC_PER_MSEC; } # include <linux/timekeeping.h> #endif |
| 1 2 1 1 1 8 7 6 2 7 1 7 2 6 8 5 5 4 2 1 2 1 1 2 2 1 2 5 32920 32886 32920 3 2 2 1 10 4 1 1 1 7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 | // SPDX-License-Identifier: GPL-2.0-only /* * xsave/xrstor support. * * Author: Suresh Siddha <suresh.b.siddha@intel.com> */ #include <linux/bitops.h> #include <linux/compat.h> #include <linux/cpu.h> #include <linux/mman.h> #include <linux/nospec.h> #include <linux/pkeys.h> #include <linux/seq_file.h> #include <linux/proc_fs.h> #include <linux/vmalloc.h> #include <linux/coredump.h> #include <asm/fpu/api.h> #include <asm/fpu/regset.h> #include <asm/fpu/signal.h> #include <asm/fpu/xcr.h> #include <asm/tlbflush.h> #include <asm/prctl.h> #include <asm/elf.h> #include <uapi/asm/elf.h> #include "context.h" #include "internal.h" #include "legacy.h" #include "xstate.h" #define for_each_extended_xfeature(bit, mask) \ (bit) = FIRST_EXTENDED_XFEATURE; \ for_each_set_bit_from(bit, (unsigned long *)&(mask), 8 * sizeof(mask)) /* * Although we spell it out in here, the Processor Trace * xfeature is completely unused. We use other mechanisms * to save/restore PT state in Linux. */ static const char *xfeature_names[] = { "x87 floating point registers", "SSE registers", "AVX registers", "MPX bounds registers", "MPX CSR", "AVX-512 opmask", "AVX-512 Hi256", "AVX-512 ZMM_Hi256", "Processor Trace (unused)", "Protection Keys User registers", "PASID state", "Control-flow User registers", "Control-flow Kernel registers (unused)", "unknown xstate feature", "unknown xstate feature", "unknown xstate feature", "unknown xstate feature", "AMX Tile config", "AMX Tile data", "unknown xstate feature", }; static unsigned short xsave_cpuid_features[] __initdata = { [XFEATURE_FP] = X86_FEATURE_FPU, [XFEATURE_SSE] = X86_FEATURE_XMM, [XFEATURE_YMM] = X86_FEATURE_AVX, [XFEATURE_BNDREGS] = X86_FEATURE_MPX, [XFEATURE_BNDCSR] = X86_FEATURE_MPX, [XFEATURE_OPMASK] = X86_FEATURE_AVX512F, [XFEATURE_ZMM_Hi256] = X86_FEATURE_AVX512F, [XFEATURE_Hi16_ZMM] = X86_FEATURE_AVX512F, [XFEATURE_PT_UNIMPLEMENTED_SO_FAR] = X86_FEATURE_INTEL_PT, [XFEATURE_PKRU] = X86_FEATURE_OSPKE, [XFEATURE_PASID] = X86_FEATURE_ENQCMD, [XFEATURE_CET_USER] = X86_FEATURE_SHSTK, [XFEATURE_XTILE_CFG] = X86_FEATURE_AMX_TILE, [XFEATURE_XTILE_DATA] = X86_FEATURE_AMX_TILE, }; static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init = { [ 0 ... XFEATURE_MAX - 1] = -1}; static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init = { [ 0 ... XFEATURE_MAX - 1] = -1}; static unsigned int xstate_flags[XFEATURE_MAX] __ro_after_init; #define XSTATE_FLAG_SUPERVISOR BIT(0) #define XSTATE_FLAG_ALIGNED64 BIT(1) /* * Return whether the system supports a given xfeature. * * Also return the name of the (most advanced) feature that the caller requested: */ int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name) { u64 xfeatures_missing = xfeatures_needed & ~fpu_kernel_cfg.max_features; if (unlikely(feature_name)) { long xfeature_idx, max_idx; u64 xfeatures_print; /* * So we use FLS here to be able to print the most advanced * feature that was requested but is missing. So if a driver * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the * missing AVX feature - this is the most informative message * to users: */ if (xfeatures_missing) xfeatures_print = xfeatures_missing; else xfeatures_print = xfeatures_needed; xfeature_idx = fls64(xfeatures_print)-1; max_idx = ARRAY_SIZE(xfeature_names)-1; xfeature_idx = min(xfeature_idx, max_idx); *feature_name = xfeature_names[xfeature_idx]; } if (xfeatures_missing) return 0; return 1; } EXPORT_SYMBOL_GPL(cpu_has_xfeatures); static bool xfeature_is_aligned64(int xfeature_nr) { return xstate_flags[xfeature_nr] & XSTATE_FLAG_ALIGNED64; } static bool xfeature_is_supervisor(int xfeature_nr) { return xstate_flags[xfeature_nr] & XSTATE_FLAG_SUPERVISOR; } static unsigned int xfeature_get_offset(u64 xcomp_bv, int xfeature) { unsigned int offs, i; /* * Non-compacted format and legacy features use the cached fixed * offsets. */ if (!cpu_feature_enabled(X86_FEATURE_XCOMPACTED) || xfeature <= XFEATURE_SSE) return xstate_offsets[xfeature]; /* * Compacted format offsets depend on the actual content of the * compacted xsave area which is determined by the xcomp_bv header * field. */ offs = FXSAVE_SIZE + XSAVE_HDR_SIZE; for_each_extended_xfeature(i, xcomp_bv) { if (xfeature_is_aligned64(i)) offs = ALIGN(offs, 64); if (i == xfeature) break; offs += xstate_sizes[i]; } return offs; } /* * Enable the extended processor state save/restore feature. * Called once per CPU onlining. */ void fpu__init_cpu_xstate(void) { if (!boot_cpu_has(X86_FEATURE_XSAVE) || !fpu_kernel_cfg.max_features) return; cr4_set_bits(X86_CR4_OSXSAVE); /* * Must happen after CR4 setup and before xsetbv() to allow KVM * lazy passthrough. Write independent of the dynamic state static * key as that does not work on the boot CPU. This also ensures * that any stale state is wiped out from XFD. Reset the per CPU * xfd cache too. */ if (cpu_feature_enabled(X86_FEATURE_XFD)) xfd_set_state(init_fpstate.xfd); /* * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features * managed by XSAVE{C, OPT, S} and XRSTOR{S}. Only XSAVE user * states can be set here. */ xsetbv(XCR_XFEATURE_ENABLED_MASK, fpu_user_cfg.max_features); /* * MSR_IA32_XSS sets supervisor states managed by XSAVES. */ if (boot_cpu_has(X86_FEATURE_XSAVES)) { wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | xfeatures_mask_independent()); } } static bool xfeature_enabled(enum xfeature xfeature) { return fpu_kernel_cfg.max_features & BIT_ULL(xfeature); } /* * Record the offsets and sizes of various xstates contained * in the XSAVE state memory layout. */ static void __init setup_xstate_cache(void) { u32 eax, ebx, ecx, edx, i; /* start at the beginning of the "extended state" */ unsigned int last_good_offset = offsetof(struct xregs_state, extended_state_area); /* * The FP xstates and SSE xstates are legacy states. They are always * in the fixed offsets in the xsave area in either compacted form * or standard form. */ xstate_offsets[XFEATURE_FP] = 0; xstate_sizes[XFEATURE_FP] = offsetof(struct fxregs_state, xmm_space); xstate_offsets[XFEATURE_SSE] = xstate_sizes[XFEATURE_FP]; xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state, xmm_space); for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); xstate_sizes[i] = eax; xstate_flags[i] = ecx; /* * If an xfeature is supervisor state, the offset in EBX is * invalid, leave it to -1. */ if (xfeature_is_supervisor(i)) continue; xstate_offsets[i] = ebx; /* * In our xstate size checks, we assume that the highest-numbered * xstate feature has the highest offset in the buffer. Ensure * it does. */ WARN_ONCE(last_good_offset > xstate_offsets[i], "x86/fpu: misordered xstate at %d\n", last_good_offset); last_good_offset = xstate_offsets[i]; } } static void __init print_xstate_feature(u64 xstate_mask) { const char *feature_name; if (cpu_has_xfeatures(xstate_mask, &feature_name)) pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name); } /* * Print out all the supported xstate features: */ static void __init print_xstate_features(void) { print_xstate_feature(XFEATURE_MASK_FP); print_xstate_feature(XFEATURE_MASK_SSE); print_xstate_feature(XFEATURE_MASK_YMM); print_xstate_feature(XFEATURE_MASK_BNDREGS); print_xstate_feature(XFEATURE_MASK_BNDCSR); print_xstate_feature(XFEATURE_MASK_OPMASK); print_xstate_feature(XFEATURE_MASK_ZMM_Hi256); print_xstate_feature(XFEATURE_MASK_Hi16_ZMM); print_xstate_feature(XFEATURE_MASK_PKRU); print_xstate_feature(XFEATURE_MASK_PASID); print_xstate_feature(XFEATURE_MASK_CET_USER); print_xstate_feature(XFEATURE_MASK_XTILE_CFG); print_xstate_feature(XFEATURE_MASK_XTILE_DATA); } /* * This check is important because it is easy to get XSTATE_* * confused with XSTATE_BIT_*. */ #define CHECK_XFEATURE(nr) do { \ WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \ WARN_ON(nr >= XFEATURE_MAX); \ } while (0) /* * Print out xstate component offsets and sizes */ static void __init print_xstate_offset_size(void) { int i; for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n", i, xfeature_get_offset(fpu_kernel_cfg.max_features, i), i, xstate_sizes[i]); } } /* * This function is called only during boot time when x86 caps are not set * up and alternative can not be used yet. */ static __init void os_xrstor_booting(struct xregs_state *xstate) { u64 mask = fpu_kernel_cfg.max_features & XFEATURE_MASK_FPSTATE; u32 lmask = mask; u32 hmask = mask >> 32; int err; if (cpu_feature_enabled(X86_FEATURE_XSAVES)) XSTATE_OP(XRSTORS, xstate, lmask, hmask, err); else XSTATE_OP(XRSTOR, xstate, lmask, hmask, err); /* * We should never fault when copying from a kernel buffer, and the FPU * state we set at boot time should be valid. */ WARN_ON_FPU(err); } /* * All supported features have either init state all zeros or are * handled in setup_init_fpu() individually. This is an explicit * feature list and does not use XFEATURE_MASK*SUPPORTED to catch * newly added supported features at build time and make people * actually look at the init state for the new feature. */ #define XFEATURES_INIT_FPSTATE_HANDLED \ (XFEATURE_MASK_FP | \ XFEATURE_MASK_SSE | \ XFEATURE_MASK_YMM | \ XFEATURE_MASK_OPMASK | \ XFEATURE_MASK_ZMM_Hi256 | \ XFEATURE_MASK_Hi16_ZMM | \ XFEATURE_MASK_PKRU | \ XFEATURE_MASK_BNDREGS | \ XFEATURE_MASK_BNDCSR | \ XFEATURE_MASK_PASID | \ XFEATURE_MASK_CET_USER | \ XFEATURE_MASK_XTILE) /* * setup the xstate image representing the init state */ static void __init setup_init_fpu_buf(void) { BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED | XFEATURE_MASK_SUPERVISOR_SUPPORTED) != XFEATURES_INIT_FPSTATE_HANDLED); if (!boot_cpu_has(X86_FEATURE_XSAVE)) return; print_xstate_features(); xstate_init_xcomp_bv(&init_fpstate.regs.xsave, init_fpstate.xfeatures); /* * Init all the features state with header.xfeatures being 0x0 */ os_xrstor_booting(&init_fpstate.regs.xsave); /* * All components are now in init state. Read the state back so * that init_fpstate contains all non-zero init state. This only * works with XSAVE, but not with XSAVEOPT and XSAVEC/S because * those use the init optimization which skips writing data for * components in init state. * * XSAVE could be used, but that would require to reshuffle the * data when XSAVEC/S is available because XSAVEC/S uses xstate * compaction. But doing so is a pointless exercise because most * components have an all zeros init state except for the legacy * ones (FP and SSE). Those can be saved with FXSAVE into the * legacy area. Adding new features requires to ensure that init * state is all zeroes or if not to add the necessary handling * here. */ fxsave(&init_fpstate.regs.fxsave); } int xfeature_size(int xfeature_nr) { u32 eax, ebx, ecx, edx; CHECK_XFEATURE(xfeature_nr); cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx); return eax; } /* Validate an xstate header supplied by userspace (ptrace or sigreturn) */ static int validate_user_xstate_header(const struct xstate_header *hdr, struct fpstate *fpstate) { /* No unknown or supervisor features may be set */ if (hdr->xfeatures & ~fpstate->user_xfeatures) return -EINVAL; /* Userspace must use the uncompacted format */ if (hdr->xcomp_bv) return -EINVAL; /* * If 'reserved' is shrunken to add a new field, make sure to validate * that new field here! */ BUILD_BUG_ON(sizeof(hdr->reserved) != 48); /* No reserved bits may be set */ if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved))) return -EINVAL; return 0; } static void __init __xstate_dump_leaves(void) { int i; u32 eax, ebx, ecx, edx; static int should_dump = 1; if (!should_dump) return; should_dump = 0; /* * Dump out a few leaves past the ones that we support * just in case there are some goodies up there */ for (i = 0; i < XFEATURE_MAX + 10; i++) { cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx); pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n", XSTATE_CPUID, i, eax, ebx, ecx, edx); } } #define XSTATE_WARN_ON(x, fmt, ...) do { \ if (WARN_ONCE(x, "XSAVE consistency problem: " fmt, ##__VA_ARGS__)) { \ __xstate_dump_leaves(); \ } \ } while (0) #define XCHECK_SZ(sz, nr, __struct) ({ \ if (WARN_ONCE(sz != sizeof(__struct), \ "[%s]: struct is %zu bytes, cpu state %d bytes\n", \ xfeature_names[nr], sizeof(__struct), sz)) { \ __xstate_dump_leaves(); \ } \ true; \ }) /** * check_xtile_data_against_struct - Check tile data state size. * * Calculate the state size by multiplying the single tile size which is * recorded in a C struct, and the number of tiles that the CPU informs. * Compare the provided size with the calculation. * * @size: The tile data state size * * Returns: 0 on success, -EINVAL on mismatch. */ static int __init check_xtile_data_against_struct(int size) { u32 max_palid, palid, state_size; u32 eax, ebx, ecx, edx; u16 max_tile; /* * Check the maximum palette id: * eax: the highest numbered palette subleaf. */ cpuid_count(TILE_CPUID, 0, &max_palid, &ebx, &ecx, &edx); /* * Cross-check each tile size and find the maximum number of * supported tiles. */ for (palid = 1, max_tile = 0; palid <= max_palid; palid++) { u16 tile_size, max; /* * Check the tile size info: * eax[31:16]: bytes per title * ebx[31:16]: the max names (or max number of tiles) */ cpuid_count(TILE_CPUID, palid, &eax, &ebx, &edx, &edx); tile_size = eax >> 16; max = ebx >> 16; if (tile_size != sizeof(struct xtile_data)) { pr_err("%s: struct is %zu bytes, cpu xtile %d bytes\n", __stringify(XFEATURE_XTILE_DATA), sizeof(struct xtile_data), tile_size); __xstate_dump_leaves(); return -EINVAL; } if (max > max_tile) max_tile = max; } state_size = sizeof(struct xtile_data) * max_tile; if (size != state_size) { pr_err("%s: calculated size is %u bytes, cpu state %d bytes\n", __stringify(XFEATURE_XTILE_DATA), state_size, size); __xstate_dump_leaves(); return -EINVAL; } return 0; } /* * We have a C struct for each 'xstate'. We need to ensure * that our software representation matches what the CPU * tells us about the state's size. */ static bool __init check_xstate_against_struct(int nr) { /* * Ask the CPU for the size of the state. */ int sz = xfeature_size(nr); /* * Match each CPU state with the corresponding software * structure. */ switch (nr) { case XFEATURE_YMM: return XCHECK_SZ(sz, nr, struct ymmh_struct); case XFEATURE_BNDREGS: return XCHECK_SZ(sz, nr, struct mpx_bndreg_state); case XFEATURE_BNDCSR: return XCHECK_SZ(sz, nr, struct mpx_bndcsr_state); case XFEATURE_OPMASK: return XCHECK_SZ(sz, nr, struct avx_512_opmask_state); case XFEATURE_ZMM_Hi256: return XCHECK_SZ(sz, nr, struct avx_512_zmm_uppers_state); case XFEATURE_Hi16_ZMM: return XCHECK_SZ(sz, nr, struct avx_512_hi16_state); case XFEATURE_PKRU: return XCHECK_SZ(sz, nr, struct pkru_state); case XFEATURE_PASID: return XCHECK_SZ(sz, nr, struct ia32_pasid_state); case XFEATURE_XTILE_CFG: return XCHECK_SZ(sz, nr, struct xtile_cfg); case XFEATURE_CET_USER: return XCHECK_SZ(sz, nr, struct cet_user_state); case XFEATURE_XTILE_DATA: check_xtile_data_against_struct(sz); return true; default: XSTATE_WARN_ON(1, "No structure for xstate: %d\n", nr); return false; } return true; } static unsigned int xstate_calculate_size(u64 xfeatures, bool compacted) { unsigned int topmost = fls64(xfeatures) - 1; unsigned int offset = xstate_offsets[topmost]; if (topmost <= XFEATURE_SSE) return sizeof(struct xregs_state); if (compacted) offset = xfeature_get_offset(xfeatures, topmost); return offset + xstate_sizes[topmost]; } /* * This essentially double-checks what the cpu told us about * how large the XSAVE buffer needs to be. We are recalculating * it to be safe. * * Independent XSAVE features allocate their own buffers and are not * covered by these checks. Only the size of the buffer for task->fpu * is checked here. */ static bool __init paranoid_xstate_size_valid(unsigned int kernel_size) { bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED); bool xsaves = cpu_feature_enabled(X86_FEATURE_XSAVES); unsigned int size = FXSAVE_SIZE + XSAVE_HDR_SIZE; int i; for_each_extended_xfeature(i, fpu_kernel_cfg.max_features) { if (!check_xstate_against_struct(i)) return false; /* * Supervisor state components can be managed only by * XSAVES. */ if (!xsaves && xfeature_is_supervisor(i)) { XSTATE_WARN_ON(1, "Got supervisor feature %d, but XSAVES not advertised\n", i); return false; } } size = xstate_calculate_size(fpu_kernel_cfg.max_features, compacted); XSTATE_WARN_ON(size != kernel_size, "size %u != kernel_size %u\n", size, kernel_size); return size == kernel_size; } /* * Get total size of enabled xstates in XCR0 | IA32_XSS. * * Note the SDM's wording here. "sub-function 0" only enumerates * the size of the *user* states. If we use it to size a buffer * that we use 'XSAVES' on, we could potentially overflow the * buffer because 'XSAVES' saves system states too. * * This also takes compaction into account. So this works for * XSAVEC as well. */ static unsigned int __init get_compacted_size(void) { unsigned int eax, ebx, ecx, edx; /* * - CPUID function 0DH, sub-function 1: * EBX enumerates the size (in bytes) required by * the XSAVES instruction for an XSAVE area * containing all the state components * corresponding to bits currently set in * XCR0 | IA32_XSS. * * When XSAVES is not available but XSAVEC is (virt), then there * are no supervisor states, but XSAVEC still uses compacted * format. */ cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); return ebx; } /* * Get the total size of the enabled xstates without the independent supervisor * features. */ static unsigned int __init get_xsave_compacted_size(void) { u64 mask = xfeatures_mask_independent(); unsigned int size; if (!mask) return get_compacted_size(); /* Disable independent features. */ wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor()); /* * Ask the hardware what size is required of the buffer. * This is the size required for the task->fpu buffer. */ size = get_compacted_size(); /* Re-enable independent features so XSAVES will work on them again. */ wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask); return size; } static unsigned int __init get_xsave_size_user(void) { unsigned int eax, ebx, ecx, edx; /* * - CPUID function 0DH, sub-function 0: * EBX enumerates the size (in bytes) required by * the XSAVE instruction for an XSAVE area * containing all the *user* state components * corresponding to bits currently set in XCR0. */ cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); return ebx; } static int __init init_xstate_size(void) { /* Recompute the context size for enabled features: */ unsigned int user_size, kernel_size, kernel_default_size; bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED); /* Uncompacted user space size */ user_size = get_xsave_size_user(); /* * XSAVES kernel size includes supervisor states and uses compacted * format. XSAVEC uses compacted format, but does not save * supervisor states. * * XSAVE[OPT] do not support supervisor states so kernel and user * size is identical. */ if (compacted) kernel_size = get_xsave_compacted_size(); else kernel_size = user_size; kernel_default_size = xstate_calculate_size(fpu_kernel_cfg.default_features, compacted); if (!paranoid_xstate_size_valid(kernel_size)) return -EINVAL; fpu_kernel_cfg.max_size = kernel_size; fpu_user_cfg.max_size = user_size; fpu_kernel_cfg.default_size = kernel_default_size; fpu_user_cfg.default_size = xstate_calculate_size(fpu_user_cfg.default_features, false); return 0; } /* * We enabled the XSAVE hardware, but something went wrong and * we can not use it. Disable it. */ static void __init fpu__init_disable_system_xstate(unsigned int legacy_size) { fpu_kernel_cfg.max_features = 0; cr4_clear_bits(X86_CR4_OSXSAVE); setup_clear_cpu_cap(X86_FEATURE_XSAVE); /* Restore the legacy size.*/ fpu_kernel_cfg.max_size = legacy_size; fpu_kernel_cfg.default_size = legacy_size; fpu_user_cfg.max_size = legacy_size; fpu_user_cfg.default_size = legacy_size; /* * Prevent enabling the static branch which enables writes to the * XFD MSR. */ init_fpstate.xfd = 0; fpstate_reset(¤t->thread.fpu); } /* * Enable and initialize the xsave feature. * Called once per system bootup. */ void __init fpu__init_system_xstate(unsigned int legacy_size) { unsigned int eax, ebx, ecx, edx; u64 xfeatures; int err; int i; if (!boot_cpu_has(X86_FEATURE_FPU)) { pr_info("x86/fpu: No FPU detected\n"); return; } if (!boot_cpu_has(X86_FEATURE_XSAVE)) { pr_info("x86/fpu: x87 FPU will use %s\n", boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE"); return; } if (boot_cpu_data.cpuid_level < XSTATE_CPUID) { WARN_ON_FPU(1); return; } /* * Find user xstates supported by the processor. */ cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx); fpu_kernel_cfg.max_features = eax + ((u64)edx << 32); /* * Find supervisor xstates supported by the processor. */ cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx); fpu_kernel_cfg.max_features |= ecx + ((u64)edx << 32); if ((fpu_kernel_cfg.max_features & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) { /* * This indicates that something really unexpected happened * with the enumeration. Disable XSAVE and try to continue * booting without it. This is too early to BUG(). */ pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n", fpu_kernel_cfg.max_features); goto out_disable; } fpu_kernel_cfg.independent_features = fpu_kernel_cfg.max_features & XFEATURE_MASK_INDEPENDENT; /* * Clear XSAVE features that are disabled in the normal CPUID. */ for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) { unsigned short cid = xsave_cpuid_features[i]; /* Careful: X86_FEATURE_FPU is 0! */ if ((i != XFEATURE_FP && !cid) || !boot_cpu_has(cid)) fpu_kernel_cfg.max_features &= ~BIT_ULL(i); } if (!cpu_feature_enabled(X86_FEATURE_XFD)) fpu_kernel_cfg.max_features &= ~XFEATURE_MASK_USER_DYNAMIC; if (!cpu_feature_enabled(X86_FEATURE_XSAVES)) fpu_kernel_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED; else fpu_kernel_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED | XFEATURE_MASK_SUPERVISOR_SUPPORTED; fpu_user_cfg.max_features = fpu_kernel_cfg.max_features; fpu_user_cfg.max_features &= XFEATURE_MASK_USER_SUPPORTED; /* Clean out dynamic features from default */ fpu_kernel_cfg.default_features = fpu_kernel_cfg.max_features; fpu_kernel_cfg.default_features &= ~XFEATURE_MASK_USER_DYNAMIC; fpu_user_cfg.default_features = fpu_user_cfg.max_features; fpu_user_cfg.default_features &= ~XFEATURE_MASK_USER_DYNAMIC; /* Store it for paranoia check at the end */ xfeatures = fpu_kernel_cfg.max_features; /* * Initialize the default XFD state in initfp_state and enable the * dynamic sizing mechanism if dynamic states are available. The * static key cannot be enabled here because this runs before * jump_label_init(). This is delayed to an initcall. */ init_fpstate.xfd = fpu_user_cfg.max_features & XFEATURE_MASK_USER_DYNAMIC; /* Set up compaction feature bit */ if (cpu_feature_enabled(X86_FEATURE_XSAVEC) || cpu_feature_enabled(X86_FEATURE_XSAVES)) setup_force_cpu_cap(X86_FEATURE_XCOMPACTED); /* Enable xstate instructions to be able to continue with initialization: */ fpu__init_cpu_xstate(); /* Cache size, offset and flags for initialization */ setup_xstate_cache(); err = init_xstate_size(); if (err) goto out_disable; /* Reset the state for the current task */ fpstate_reset(¤t->thread.fpu); /* * Update info used for ptrace frames; use standard-format size and no * supervisor xstates: */ update_regset_xstate_info(fpu_user_cfg.max_size, fpu_user_cfg.max_features); /* * init_fpstate excludes dynamic states as they are large but init * state is zero. */ init_fpstate.size = fpu_kernel_cfg.default_size; init_fpstate.xfeatures = fpu_kernel_cfg.default_features; if (init_fpstate.size > sizeof(init_fpstate.regs)) { pr_warn("x86/fpu: init_fpstate buffer too small (%zu < %d), disabling XSAVE\n", sizeof(init_fpstate.regs), init_fpstate.size); goto out_disable; } setup_init_fpu_buf(); /* * Paranoia check whether something in the setup modified the * xfeatures mask. */ if (xfeatures != fpu_kernel_cfg.max_features) { pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init, disabling XSAVE\n", xfeatures, fpu_kernel_cfg.max_features); goto out_disable; } /* * CPU capabilities initialization runs before FPU init. So * X86_FEATURE_OSXSAVE is not set. Now that XSAVE is completely * functional, set the feature bit so depending code works. */ setup_force_cpu_cap(X86_FEATURE_OSXSAVE); print_xstate_offset_size(); pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n", fpu_kernel_cfg.max_features, fpu_kernel_cfg.max_size, boot_cpu_has(X86_FEATURE_XCOMPACTED) ? "compacted" : "standard"); return; out_disable: /* something went wrong, try to boot without any XSAVE support */ fpu__init_disable_system_xstate(legacy_size); } /* * Restore minimal FPU state after suspend: */ void fpu__resume_cpu(void) { /* * Restore XCR0 on xsave capable CPUs: */ if (cpu_feature_enabled(X86_FEATURE_XSAVE)) xsetbv(XCR_XFEATURE_ENABLED_MASK, fpu_user_cfg.max_features); /* * Restore IA32_XSS. The same CPUID bit enumerates support * of XSAVES and MSR_IA32_XSS. */ if (cpu_feature_enabled(X86_FEATURE_XSAVES)) { wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | xfeatures_mask_independent()); } if (fpu_state_size_dynamic()) wrmsrl(MSR_IA32_XFD, current->thread.fpu.fpstate->xfd); } /* * Given an xstate feature nr, calculate where in the xsave * buffer the state is. Callers should ensure that the buffer * is valid. */ static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr) { u64 xcomp_bv = xsave->header.xcomp_bv; if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr))) return NULL; if (cpu_feature_enabled(X86_FEATURE_XCOMPACTED)) { if (WARN_ON_ONCE(!(xcomp_bv & BIT_ULL(xfeature_nr)))) return NULL; } return (void *)xsave + xfeature_get_offset(xcomp_bv, xfeature_nr); } /* * Given the xsave area and a state inside, this function returns the * address of the state. * * This is the API that is called to get xstate address in either * standard format or compacted format of xsave area. * * Note that if there is no data for the field in the xsave buffer * this will return NULL. * * Inputs: * xstate: the thread's storage area for all FPU data * xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP, * XFEATURE_SSE, etc...) * Output: * address of the state in the xsave area, or NULL if the * field is not present in the xsave buffer. */ void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr) { /* * Do we even *have* xsave state? */ if (!boot_cpu_has(X86_FEATURE_XSAVE)) return NULL; /* * We should not ever be requesting features that we * have not enabled. */ if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr))) return NULL; /* * This assumes the last 'xsave*' instruction to * have requested that 'xfeature_nr' be saved. * If it did not, we might be seeing and old value * of the field in the buffer. * * This can happen because the last 'xsave' did not * request that this feature be saved (unlikely) * or because the "init optimization" caused it * to not be saved. */ if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr))) return NULL; return __raw_xsave_addr(xsave, xfeature_nr); } EXPORT_SYMBOL_GPL(get_xsave_addr); /* * Given an xstate feature nr, calculate where in the xsave buffer the state is. * The xsave buffer should be in standard format, not compacted (e.g. user mode * signal frames). */ void __user *get_xsave_addr_user(struct xregs_state __user *xsave, int xfeature_nr) { if (WARN_ON_ONCE(!xfeature_enabled(xfeature_nr))) return NULL; return (void __user *)xsave + xstate_offsets[xfeature_nr]; } #ifdef CONFIG_ARCH_HAS_PKEYS /* * This will go out and modify PKRU register to set the access * rights for @pkey to @init_val. */ int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, unsigned long init_val) { u32 old_pkru, new_pkru_bits = 0; int pkey_shift; /* * This check implies XSAVE support. OSPKE only gets * set if we enable XSAVE and we enable PKU in XCR0. */ if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) return -EINVAL; /* * This code should only be called with valid 'pkey' * values originating from in-kernel users. Complain * if a bad value is observed. */ if (WARN_ON_ONCE(pkey >= arch_max_pkey())) return -EINVAL; /* Set the bits we need in PKRU: */ if (init_val & PKEY_DISABLE_ACCESS) new_pkru_bits |= PKRU_AD_BIT; if (init_val & PKEY_DISABLE_WRITE) new_pkru_bits |= PKRU_WD_BIT; /* Shift the bits in to the correct place in PKRU for pkey: */ pkey_shift = pkey * PKRU_BITS_PER_PKEY; new_pkru_bits <<= pkey_shift; /* Get old PKRU and mask off any old bits in place: */ old_pkru = read_pkru(); old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift); /* Write old part along with new part: */ write_pkru(old_pkru | new_pkru_bits); return 0; } #endif /* ! CONFIG_ARCH_HAS_PKEYS */ static void copy_feature(bool from_xstate, struct membuf *to, void *xstate, void *init_xstate, unsigned int size) { membuf_write(to, from_xstate ? xstate : init_xstate, size); } /** * __copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer * @to: membuf descriptor * @fpstate: The fpstate buffer from which to copy * @xfeatures: The mask of xfeatures to save (XSAVE mode only) * @pkru_val: The PKRU value to store in the PKRU component * @copy_mode: The requested copy mode * * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming * format, i.e. from the kernel internal hardware dependent storage format * to the requested @mode. UABI XSTATE is always uncompacted! * * It supports partial copy but @to.pos always starts from zero. */ void __copy_xstate_to_uabi_buf(struct membuf to, struct fpstate *fpstate, u64 xfeatures, u32 pkru_val, enum xstate_copy_mode copy_mode) { const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr); struct xregs_state *xinit = &init_fpstate.regs.xsave; struct xregs_state *xsave = &fpstate->regs.xsave; struct xstate_header header; unsigned int zerofrom; u64 mask; int i; memset(&header, 0, sizeof(header)); header.xfeatures = xsave->header.xfeatures; /* Mask out the feature bits depending on copy mode */ switch (copy_mode) { case XSTATE_COPY_FP: header.xfeatures &= XFEATURE_MASK_FP; break; case XSTATE_COPY_FX: header.xfeatures &= XFEATURE_MASK_FP | XFEATURE_MASK_SSE; break; case XSTATE_COPY_XSAVE: header.xfeatures &= fpstate->user_xfeatures & xfeatures; break; } /* Copy FP state up to MXCSR */ copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387, &xinit->i387, off_mxcsr); /* Copy MXCSR when SSE or YMM are set in the feature mask */ copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM), &to, &xsave->i387.mxcsr, &xinit->i387.mxcsr, MXCSR_AND_FLAGS_SIZE); /* Copy the remaining FP state */ copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387.st_space, &xinit->i387.st_space, sizeof(xsave->i387.st_space)); /* Copy the SSE state - shared with YMM, but independently managed */ copy_feature(header.xfeatures & XFEATURE_MASK_SSE, &to, &xsave->i387.xmm_space, &xinit->i387.xmm_space, sizeof(xsave->i387.xmm_space)); if (copy_mode != XSTATE_COPY_XSAVE) goto out; /* Zero the padding area */ membuf_zero(&to, sizeof(xsave->i387.padding)); /* Copy xsave->i387.sw_reserved */ membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved)); /* Copy the user space relevant state of @xsave->header */ membuf_write(&to, &header, sizeof(header)); zerofrom = offsetof(struct xregs_state, extended_state_area); /* * This 'mask' indicates which states to copy from fpstate. * Those extended states that are not present in fpstate are * either disabled or initialized: * * In non-compacted format, disabled features still occupy * state space but there is no state to copy from in the * compacted init_fpstate. The gap tracking will zero these * states. * * The extended features have an all zeroes init state. Thus, * remove them from 'mask' to zero those features in the user * buffer instead of retrieving them from init_fpstate. */ mask = header.xfeatures; for_each_extended_xfeature(i, mask) { /* * If there was a feature or alignment gap, zero the space * in the destination buffer. */ if (zerofrom < xstate_offsets[i]) membuf_zero(&to, xstate_offsets[i] - zerofrom); if (i == XFEATURE_PKRU) { struct pkru_state pkru = {0}; /* * PKRU is not necessarily up to date in the * XSAVE buffer. Use the provided value. */ pkru.pkru = pkru_val; membuf_write(&to, &pkru, sizeof(pkru)); } else { membuf_write(&to, __raw_xsave_addr(xsave, i), xstate_sizes[i]); } /* * Keep track of the last copied state in the non-compacted * target buffer for gap zeroing. */ zerofrom = xstate_offsets[i] + xstate_sizes[i]; } out: if (to.left) membuf_zero(&to, to.left); } /** * copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer * @to: membuf descriptor * @tsk: The task from which to copy the saved xstate * @copy_mode: The requested copy mode * * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming * format, i.e. from the kernel internal hardware dependent storage format * to the requested @mode. UABI XSTATE is always uncompacted! * * It supports partial copy but @to.pos always starts from zero. */ void copy_xstate_to_uabi_buf(struct membuf to, struct task_struct *tsk, enum xstate_copy_mode copy_mode) { __copy_xstate_to_uabi_buf(to, tsk->thread.fpu.fpstate, tsk->thread.fpu.fpstate->user_xfeatures, tsk->thread.pkru, copy_mode); } static int copy_from_buffer(void *dst, unsigned int offset, unsigned int size, const void *kbuf, const void __user *ubuf) { if (kbuf) { memcpy(dst, kbuf + offset, size); } else { if (copy_from_user(dst, ubuf + offset, size)) return -EFAULT; } return 0; } /** * copy_uabi_to_xstate - Copy a UABI format buffer to the kernel xstate * @fpstate: The fpstate buffer to copy to * @kbuf: The UABI format buffer, if it comes from the kernel * @ubuf: The UABI format buffer, if it comes from userspace * @pkru: The location to write the PKRU value to * * Converts from the UABI format into the kernel internal hardware * dependent format. * * This function ultimately has three different callers with distinct PKRU * behavior. * 1. When called from sigreturn the PKRU register will be restored from * @fpstate via an XRSTOR. Correctly copying the UABI format buffer to * @fpstate is sufficient to cover this case, but the caller will also * pass a pointer to the thread_struct's pkru field in @pkru and updating * it is harmless. * 2. When called from ptrace the PKRU register will be restored from the * thread_struct's pkru field. A pointer to that is passed in @pkru. * The kernel will restore it manually, so the XRSTOR behavior that resets * the PKRU register to the hardware init value (0) if the corresponding * xfeatures bit is not set is emulated here. * 3. When called from KVM the PKRU register will be restored from the vcpu's * pkru field. A pointer to that is passed in @pkru. KVM hasn't used * XRSTOR and hasn't had the PKRU resetting behavior described above. To * preserve that KVM behavior, it passes NULL for @pkru if the xfeatures * bit is not set. */ static int copy_uabi_to_xstate(struct fpstate *fpstate, const void *kbuf, const void __user *ubuf, u32 *pkru) { struct xregs_state *xsave = &fpstate->regs.xsave; unsigned int offset, size; struct xstate_header hdr; u64 mask; int i; offset = offsetof(struct xregs_state, header); if (copy_from_buffer(&hdr, offset, sizeof(hdr), kbuf, ubuf)) return -EFAULT; if (validate_user_xstate_header(&hdr, fpstate)) return -EINVAL; /* Validate MXCSR when any of the related features is in use */ mask = XFEATURE_MASK_FP | XFEATURE_MASK_SSE | XFEATURE_MASK_YMM; if (hdr.xfeatures & mask) { u32 mxcsr[2]; offset = offsetof(struct fxregs_state, mxcsr); if (copy_from_buffer(mxcsr, offset, sizeof(mxcsr), kbuf, ubuf)) return -EFAULT; /* Reserved bits in MXCSR must be zero. */ if (mxcsr[0] & ~mxcsr_feature_mask) return -EINVAL; /* SSE and YMM require MXCSR even when FP is not in use. */ if (!(hdr.xfeatures & XFEATURE_MASK_FP)) { xsave->i387.mxcsr = mxcsr[0]; xsave->i387.mxcsr_mask = mxcsr[1]; } } for (i = 0; i < XFEATURE_MAX; i++) { mask = BIT_ULL(i); if (hdr.xfeatures & mask) { void *dst = __raw_xsave_addr(xsave, i); offset = xstate_offsets[i]; size = xstate_sizes[i]; if (copy_from_buffer(dst, offset, size, kbuf, ubuf)) return -EFAULT; } } if (hdr.xfeatures & XFEATURE_MASK_PKRU) { struct pkru_state *xpkru; xpkru = __raw_xsave_addr(xsave, XFEATURE_PKRU); *pkru = xpkru->pkru; } else { /* * KVM may pass NULL here to indicate that it does not need * PKRU updated. */ if (pkru) *pkru = 0; } /* * The state that came in from userspace was user-state only. * Mask all the user states out of 'xfeatures': */ xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL; /* * Add back in the features that came in from userspace: */ xsave->header.xfeatures |= hdr.xfeatures; return 0; } /* * Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S] * format and copy to the target thread. Used by ptrace and KVM. */ int copy_uabi_from_kernel_to_xstate(struct fpstate *fpstate, const void *kbuf, u32 *pkru) { return copy_uabi_to_xstate(fpstate, kbuf, NULL, pkru); } /* * Convert from a sigreturn standard-format user-space buffer to kernel * XSAVE[S] format and copy to the target thread. This is called from the * sigreturn() and rt_sigreturn() system calls. */ int copy_sigframe_from_user_to_xstate(struct task_struct *tsk, const void __user *ubuf) { return copy_uabi_to_xstate(tsk->thread.fpu.fpstate, NULL, ubuf, &tsk->thread.pkru); } static bool validate_independent_components(u64 mask) { u64 xchk; if (WARN_ON_FPU(!cpu_feature_enabled(X86_FEATURE_XSAVES))) return false; xchk = ~xfeatures_mask_independent(); if (WARN_ON_ONCE(!mask || mask & xchk)) return false; return true; } /** * xsaves - Save selected components to a kernel xstate buffer * @xstate: Pointer to the buffer * @mask: Feature mask to select the components to save * * The @xstate buffer must be 64 byte aligned and correctly initialized as * XSAVES does not write the full xstate header. Before first use the * buffer should be zeroed otherwise a consecutive XRSTORS from that buffer * can #GP. * * The feature mask must be a subset of the independent features. */ void xsaves(struct xregs_state *xstate, u64 mask) { int err; if (!validate_independent_components(mask)) return; XSTATE_OP(XSAVES, xstate, (u32)mask, (u32)(mask >> 32), err); WARN_ON_ONCE(err); } /** * xrstors - Restore selected components from a kernel xstate buffer * @xstate: Pointer to the buffer * @mask: Feature mask to select the components to restore * * The @xstate buffer must be 64 byte aligned and correctly initialized * otherwise XRSTORS from that buffer can #GP. * * Proper usage is to restore the state which was saved with * xsaves() into @xstate. * * The feature mask must be a subset of the independent features. */ void xrstors(struct xregs_state *xstate, u64 mask) { int err; if (!validate_independent_components(mask)) return; XSTATE_OP(XRSTORS, xstate, (u32)mask, (u32)(mask >> 32), err); WARN_ON_ONCE(err); } #if IS_ENABLED(CONFIG_KVM) void fpstate_clear_xstate_component(struct fpstate *fps, unsigned int xfeature) { void *addr = get_xsave_addr(&fps->regs.xsave, xfeature); if (addr) memset(addr, 0, xstate_sizes[xfeature]); } EXPORT_SYMBOL_GPL(fpstate_clear_xstate_component); #endif #ifdef CONFIG_X86_64 #ifdef CONFIG_X86_DEBUG_FPU /* * Ensure that a subsequent XSAVE* or XRSTOR* instruction with RFBM=@mask * can safely operate on the @fpstate buffer. */ static bool xstate_op_valid(struct fpstate *fpstate, u64 mask, bool rstor) { u64 xfd = __this_cpu_read(xfd_state); if (fpstate->xfd == xfd) return true; /* * The XFD MSR does not match fpstate->xfd. That's invalid when * the passed in fpstate is current's fpstate. */ if (fpstate->xfd == current->thread.fpu.fpstate->xfd) return false; /* * XRSTOR(S) from init_fpstate are always correct as it will just * bring all components into init state and not read from the * buffer. XSAVE(S) raises #PF after init. */ if (fpstate == &init_fpstate) return rstor; /* * XSAVE(S): clone(), fpu_swap_kvm_fpstate() * XRSTORS(S): fpu_swap_kvm_fpstate() */ /* * No XSAVE/XRSTOR instructions (except XSAVE itself) touch * the buffer area for XFD-disabled state components. */ mask &= ~xfd; /* * Remove features which are valid in fpstate. They * have space allocated in fpstate. */ mask &= ~fpstate->xfeatures; /* * Any remaining state components in 'mask' might be written * by XSAVE/XRSTOR. Fail validation it found. */ return !mask; } void xfd_validate_state(struct fpstate *fpstate, u64 mask, bool rstor) { WARN_ON_ONCE(!xstate_op_valid(fpstate, mask, rstor)); } #endif /* CONFIG_X86_DEBUG_FPU */ static int __init xfd_update_static_branch(void) { /* * If init_fpstate.xfd has bits set then dynamic features are * available and the dynamic sizing must be enabled. */ if (init_fpstate.xfd) static_branch_enable(&__fpu_state_size_dynamic); return 0; } arch_initcall(xfd_update_static_branch) void fpstate_free(struct fpu *fpu) { if (fpu->fpstate && fpu->fpstate != &fpu->__fpstate) vfree(fpu->fpstate); } /** * fpstate_realloc - Reallocate struct fpstate for the requested new features * * @xfeatures: A bitmap of xstate features which extend the enabled features * of that task * @ksize: The required size for the kernel buffer * @usize: The required size for user space buffers * @guest_fpu: Pointer to a guest FPU container. NULL for host allocations * * Note vs. vmalloc(): If the task with a vzalloc()-allocated buffer * terminates quickly, vfree()-induced IPIs may be a concern, but tasks * with large states are likely to live longer. * * Returns: 0 on success, -ENOMEM on allocation error. */ static int fpstate_realloc(u64 xfeatures, unsigned int ksize, unsigned int usize, struct fpu_guest *guest_fpu) { struct fpu *fpu = ¤t->thread.fpu; struct fpstate *curfps, *newfps = NULL; unsigned int fpsize; bool in_use; fpsize = ksize + ALIGN(offsetof(struct fpstate, regs), 64); newfps = vzalloc(fpsize); if (!newfps) return -ENOMEM; newfps->size = ksize; newfps->user_size = usize; newfps->is_valloc = true; /* * When a guest FPU is supplied, use @guest_fpu->fpstate * as reference independent whether it is in use or not. */ curfps = guest_fpu ? guest_fpu->fpstate : fpu->fpstate; /* Determine whether @curfps is the active fpstate */ in_use = fpu->fpstate == curfps; if (guest_fpu) { newfps->is_guest = true; newfps->is_confidential = curfps->is_confidential; newfps->in_use = curfps->in_use; guest_fpu->xfeatures |= xfeatures; guest_fpu->uabi_size = usize; } fpregs_lock(); /* * If @curfps is in use, ensure that the current state is in the * registers before swapping fpstate as that might invalidate it * due to layout changes. */ if (in_use && test_thread_flag(TIF_NEED_FPU_LOAD)) fpregs_restore_userregs(); newfps->xfeatures = curfps->xfeatures | xfeatures; newfps->user_xfeatures = curfps->user_xfeatures | xfeatures; newfps->xfd = curfps->xfd & ~xfeatures; /* Do the final updates within the locked region */ xstate_init_xcomp_bv(&newfps->regs.xsave, newfps->xfeatures); if (guest_fpu) { guest_fpu->fpstate = newfps; /* If curfps is active, update the FPU fpstate pointer */ if (in_use) fpu->fpstate = newfps; } else { fpu->fpstate = newfps; } if (in_use) xfd_update_state(fpu->fpstate); fpregs_unlock(); /* Only free valloc'ed state */ if (curfps && curfps->is_valloc) vfree(curfps); return 0; } static int validate_sigaltstack(unsigned int usize) { struct task_struct *thread, *leader = current->group_leader; unsigned long framesize = get_sigframe_size(); lockdep_assert_held(¤t->sighand->siglock); /* get_sigframe_size() is based on fpu_user_cfg.max_size */ framesize -= fpu_user_cfg.max_size; framesize += usize; for_each_thread(leader, thread) { if (thread->sas_ss_size && thread->sas_ss_size < framesize) return -ENOSPC; } return 0; } static int __xstate_request_perm(u64 permitted, u64 requested, bool guest) { /* * This deliberately does not exclude !XSAVES as we still might * decide to optionally context switch XCR0 or talk the silicon * vendors into extending XFD for the pre AMX states, especially * AVX512. */ bool compacted = cpu_feature_enabled(X86_FEATURE_XCOMPACTED); struct fpu *fpu = ¤t->group_leader->thread.fpu; struct fpu_state_perm *perm; unsigned int ksize, usize; u64 mask; int ret = 0; /* Check whether fully enabled */ if ((permitted & requested) == requested) return 0; /* Calculate the resulting kernel state size */ mask = permitted | requested; /* Take supervisor states into account on the host */ if (!guest) mask |= xfeatures_mask_supervisor(); ksize = xstate_calculate_size(mask, compacted); /* Calculate the resulting user state size */ mask &= XFEATURE_MASK_USER_SUPPORTED; usize = xstate_calculate_size(mask, false); if (!guest) { ret = validate_sigaltstack(usize); if (ret) return ret; } perm = guest ? &fpu->guest_perm : &fpu->perm; /* Pairs with the READ_ONCE() in xstate_get_group_perm() */ WRITE_ONCE(perm->__state_perm, mask); /* Protected by sighand lock */ perm->__state_size = ksize; perm->__user_state_size = usize; return ret; } /* * Permissions array to map facilities with more than one component */ static const u64 xstate_prctl_req[XFEATURE_MAX] = { [XFEATURE_XTILE_DATA] = XFEATURE_MASK_XTILE_DATA, }; static int xstate_request_perm(unsigned long idx, bool guest) { u64 permitted, requested; int ret; if (idx >= XFEATURE_MAX) return -EINVAL; /* * Look up the facility mask which can require more than * one xstate component. */ idx = array_index_nospec(idx, ARRAY_SIZE(xstate_prctl_req)); requested = xstate_prctl_req[idx]; if (!requested) return -EOPNOTSUPP; if ((fpu_user_cfg.max_features & requested) != requested) return -EOPNOTSUPP; /* Lockless quick check */ permitted = xstate_get_group_perm(guest); if ((permitted & requested) == requested) return 0; /* Protect against concurrent modifications */ spin_lock_irq(¤t->sighand->siglock); permitted = xstate_get_group_perm(guest); /* First vCPU allocation locks the permissions. */ if (guest && (permitted & FPU_GUEST_PERM_LOCKED)) ret = -EBUSY; else ret = __xstate_request_perm(permitted, requested, guest); spin_unlock_irq(¤t->sighand->siglock); return ret; } int __xfd_enable_feature(u64 xfd_err, struct fpu_guest *guest_fpu) { u64 xfd_event = xfd_err & XFEATURE_MASK_USER_DYNAMIC; struct fpu_state_perm *perm; unsigned int ksize, usize; struct fpu *fpu; if (!xfd_event) { if (!guest_fpu) pr_err_once("XFD: Invalid xfd error: %016llx\n", xfd_err); return 0; } /* Protect against concurrent modifications */ spin_lock_irq(¤t->sighand->siglock); /* If not permitted let it die */ if ((xstate_get_group_perm(!!guest_fpu) & xfd_event) != xfd_event) { spin_unlock_irq(¤t->sighand->siglock); return -EPERM; } fpu = ¤t->group_leader->thread.fpu; perm = guest_fpu ? &fpu->guest_perm : &fpu->perm; ksize = perm->__state_size; usize = perm->__user_state_size; /* * The feature is permitted. State size is sufficient. Dropping * the lock is safe here even if more features are added from * another task, the retrieved buffer sizes are valid for the * currently requested feature(s). */ spin_unlock_irq(¤t->sighand->siglock); /* * Try to allocate a new fpstate. If that fails there is no way * out. */ if (fpstate_realloc(xfd_event, ksize, usize, guest_fpu)) return -EFAULT; return 0; } int xfd_enable_feature(u64 xfd_err) { return __xfd_enable_feature(xfd_err, NULL); } #else /* CONFIG_X86_64 */ static inline int xstate_request_perm(unsigned long idx, bool guest) { return -EPERM; } #endif /* !CONFIG_X86_64 */ u64 xstate_get_guest_group_perm(void) { return xstate_get_group_perm(true); } EXPORT_SYMBOL_GPL(xstate_get_guest_group_perm); /** * fpu_xstate_prctl - xstate permission operations * @option: A subfunction of arch_prctl() * @arg2: option argument * Return: 0 if successful; otherwise, an error code * * Option arguments: * * ARCH_GET_XCOMP_SUPP: Pointer to user space u64 to store the info * ARCH_GET_XCOMP_PERM: Pointer to user space u64 to store the info * ARCH_REQ_XCOMP_PERM: Facility number requested * * For facilities which require more than one XSTATE component, the request * must be the highest state component number related to that facility, * e.g. for AMX which requires XFEATURE_XTILE_CFG(17) and * XFEATURE_XTILE_DATA(18) this would be XFEATURE_XTILE_DATA(18). */ long fpu_xstate_prctl(int option, unsigned long arg2) { u64 __user *uptr = (u64 __user *)arg2; u64 permitted, supported; unsigned long idx = arg2; bool guest = false; switch (option) { case ARCH_GET_XCOMP_SUPP: supported = fpu_user_cfg.max_features | fpu_user_cfg.legacy_features; return put_user(supported, uptr); case ARCH_GET_XCOMP_PERM: /* * Lockless snapshot as it can also change right after the * dropping the lock. */ permitted = xstate_get_host_group_perm(); permitted &= XFEATURE_MASK_USER_SUPPORTED; return put_user(permitted, uptr); case ARCH_GET_XCOMP_GUEST_PERM: permitted = xstate_get_guest_group_perm(); permitted &= XFEATURE_MASK_USER_SUPPORTED; return put_user(permitted, uptr); case ARCH_REQ_XCOMP_GUEST_PERM: guest = true; fallthrough; case ARCH_REQ_XCOMP_PERM: if (!IS_ENABLED(CONFIG_X86_64)) return -EOPNOTSUPP; return xstate_request_perm(idx, guest); default: return -EINVAL; } } #ifdef CONFIG_PROC_PID_ARCH_STATUS /* * Report the amount of time elapsed in millisecond since last AVX512 * use in the task. */ static void avx512_status(struct seq_file *m, struct task_struct *task) { unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp); long delta; if (!timestamp) { /* * Report -1 if no AVX512 usage */ delta = -1; } else { delta = (long)(jiffies - timestamp); /* * Cap to LONG_MAX if time difference > LONG_MAX */ if (delta < 0) delta = LONG_MAX; delta = jiffies_to_msecs(delta); } seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta); seq_putc(m, '\n'); } /* * Report architecture specific information */ int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns, struct pid *pid, struct task_struct *task) { /* * Report AVX512 state if the processor and build option supported. */ if (cpu_feature_enabled(X86_FEATURE_AVX512F)) avx512_status(m, task); return 0; } #endif /* CONFIG_PROC_PID_ARCH_STATUS */ #ifdef CONFIG_COREDUMP static const char owner_name[] = "LINUX"; /* * Dump type, size, offset and flag values for every xfeature that is present. */ static int dump_xsave_layout_desc(struct coredump_params *cprm) { int num_records = 0; int i; for_each_extended_xfeature(i, fpu_user_cfg.max_features) { struct x86_xfeat_component xc = { .type = i, .size = xstate_sizes[i], .offset = xstate_offsets[i], /* reserved for future use */ .flags = 0, }; if (!dump_emit(cprm, &xc, sizeof(xc))) return 0; num_records++; } return num_records; } static u32 get_xsave_desc_size(void) { u32 cnt = 0; u32 i; for_each_extended_xfeature(i, fpu_user_cfg.max_features) cnt++; return cnt * (sizeof(struct x86_xfeat_component)); } int elf_coredump_extra_notes_write(struct coredump_params *cprm) { int num_records = 0; struct elf_note en; if (!fpu_user_cfg.max_features) return 0; en.n_namesz = sizeof(owner_name); en.n_descsz = get_xsave_desc_size(); en.n_type = NT_X86_XSAVE_LAYOUT; if (!dump_emit(cprm, &en, sizeof(en))) return 1; if (!dump_emit(cprm, owner_name, en.n_namesz)) return 1; if (!dump_align(cprm, 4)) return 1; num_records = dump_xsave_layout_desc(cprm); if (!num_records) return 1; /* Total size should be equal to the number of records */ if ((sizeof(struct x86_xfeat_component) * num_records) != en.n_descsz) return 1; return 0; } int elf_coredump_extra_notes_size(void) { int size; if (!fpu_user_cfg.max_features) return 0; /* .note header */ size = sizeof(struct elf_note); /* Name plus alignment to 4 bytes */ size += roundup(sizeof(owner_name), 4); size += get_xsave_desc_size(); return size; } #endif /* CONFIG_COREDUMP */ |
| 4 5 3 5 4 10 3 1 1 1 2 2 9 3 18 18 12 12 8 4 3 15 1 14 14 1 1 5 4 2 1 159 157 159 1 161 160 7 1 2 1 2 1 1 1 7 7 7 7 7 7 11 11 11 3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 | // SPDX-License-Identifier: GPL-2.0-only /* * Minimal file system backend for holding eBPF maps and programs, * used by bpf(2) object pinning. * * Authors: * * Daniel Borkmann <daniel@iogearbox.net> */ #include <linux/init.h> #include <linux/magic.h> #include <linux/major.h> #include <linux/mount.h> #include <linux/namei.h> #include <linux/fs.h> #include <linux/fs_context.h> #include <linux/fs_parser.h> #include <linux/kdev_t.h> #include <linux/filter.h> #include <linux/bpf.h> #include <linux/bpf_trace.h> #include <linux/kstrtox.h> #include "preload/bpf_preload.h" enum bpf_type { BPF_TYPE_UNSPEC = 0, BPF_TYPE_PROG, BPF_TYPE_MAP, BPF_TYPE_LINK, }; static void *bpf_any_get(void *raw, enum bpf_type type) { switch (type) { case BPF_TYPE_PROG: bpf_prog_inc(raw); break; case BPF_TYPE_MAP: bpf_map_inc_with_uref(raw); break; case BPF_TYPE_LINK: bpf_link_inc(raw); break; default: WARN_ON_ONCE(1); break; } return raw; } static void bpf_any_put(void *raw, enum bpf_type type) { switch (type) { case BPF_TYPE_PROG: bpf_prog_put(raw); break; case BPF_TYPE_MAP: bpf_map_put_with_uref(raw); break; case BPF_TYPE_LINK: bpf_link_put(raw); break; default: WARN_ON_ONCE(1); break; } } static void *bpf_fd_probe_obj(u32 ufd, enum bpf_type *type) { void *raw; raw = bpf_map_get_with_uref(ufd); if (!IS_ERR(raw)) { *type = BPF_TYPE_MAP; return raw; } raw = bpf_prog_get(ufd); if (!IS_ERR(raw)) { *type = BPF_TYPE_PROG; return raw; } raw = bpf_link_get_from_fd(ufd); if (!IS_ERR(raw)) { *type = BPF_TYPE_LINK; return raw; } return ERR_PTR(-EINVAL); } static const struct inode_operations bpf_dir_iops; static const struct inode_operations bpf_prog_iops = { }; static const struct inode_operations bpf_map_iops = { }; static const struct inode_operations bpf_link_iops = { }; struct inode *bpf_get_inode(struct super_block *sb, const struct inode *dir, umode_t mode) { struct inode *inode; switch (mode & S_IFMT) { case S_IFDIR: case S_IFREG: case S_IFLNK: break; default: return ERR_PTR(-EINVAL); } inode = new_inode(sb); if (!inode) return ERR_PTR(-ENOSPC); inode->i_ino = get_next_ino(); simple_inode_init_ts(inode); inode_init_owner(&nop_mnt_idmap, inode, dir, mode); return inode; } static int bpf_inode_type(const struct inode *inode, enum bpf_type *type) { *type = BPF_TYPE_UNSPEC; if (inode->i_op == &bpf_prog_iops) *type = BPF_TYPE_PROG; else if (inode->i_op == &bpf_map_iops) *type = BPF_TYPE_MAP; else if (inode->i_op == &bpf_link_iops) *type = BPF_TYPE_LINK; else return -EACCES; return 0; } static void bpf_dentry_finalize(struct dentry *dentry, struct inode *inode, struct inode *dir) { d_instantiate(dentry, inode); dget(dentry); inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); } static int bpf_mkdir(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, umode_t mode) { struct inode *inode; inode = bpf_get_inode(dir->i_sb, dir, mode | S_IFDIR); if (IS_ERR(inode)) return PTR_ERR(inode); inode->i_op = &bpf_dir_iops; inode->i_fop = &simple_dir_operations; inc_nlink(inode); inc_nlink(dir); bpf_dentry_finalize(dentry, inode, dir); return 0; } struct map_iter { void *key; bool done; }; static struct map_iter *map_iter(struct seq_file *m) { return m->private; } static struct bpf_map *seq_file_to_map(struct seq_file *m) { return file_inode(m->file)->i_private; } static void map_iter_free(struct map_iter *iter) { if (iter) { kfree(iter->key); kfree(iter); } } static struct map_iter *map_iter_alloc(struct bpf_map *map) { struct map_iter *iter; iter = kzalloc(sizeof(*iter), GFP_KERNEL | __GFP_NOWARN); if (!iter) goto error; iter->key = kzalloc(map->key_size, GFP_KERNEL | __GFP_NOWARN); if (!iter->key) goto error; return iter; error: map_iter_free(iter); return NULL; } static void *map_seq_next(struct seq_file *m, void *v, loff_t *pos) { struct bpf_map *map = seq_file_to_map(m); void *key = map_iter(m)->key; void *prev_key; (*pos)++; if (map_iter(m)->done) return NULL; if (unlikely(v == SEQ_START_TOKEN)) prev_key = NULL; else prev_key = key; rcu_read_lock(); if (map->ops->map_get_next_key(map, prev_key, key)) { map_iter(m)->done = true; key = NULL; } rcu_read_unlock(); return key; } static void *map_seq_start(struct seq_file *m, loff_t *pos) { if (map_iter(m)->done) return NULL; return *pos ? map_iter(m)->key : SEQ_START_TOKEN; } static void map_seq_stop(struct seq_file *m, void *v) { } static int map_seq_show(struct seq_file *m, void *v) { struct bpf_map *map = seq_file_to_map(m); void *key = map_iter(m)->key; if (unlikely(v == SEQ_START_TOKEN)) { seq_puts(m, "# WARNING!! The output is for debug purpose only\n"); seq_puts(m, "# WARNING!! The output format will change\n"); } else { map->ops->map_seq_show_elem(map, key, m); } return 0; } static const struct seq_operations bpffs_map_seq_ops = { .start = map_seq_start, .next = map_seq_next, .show = map_seq_show, .stop = map_seq_stop, }; static int bpffs_map_open(struct inode *inode, struct file *file) { struct bpf_map *map = inode->i_private; struct map_iter *iter; struct seq_file *m; int err; iter = map_iter_alloc(map); if (!iter) return -ENOMEM; err = seq_open(file, &bpffs_map_seq_ops); if (err) { map_iter_free(iter); return err; } m = file->private_data; m->private = iter; return 0; } static int bpffs_map_release(struct inode *inode, struct file *file) { struct seq_file *m = file->private_data; map_iter_free(map_iter(m)); return seq_release(inode, file); } /* bpffs_map_fops should only implement the basic * read operation for a BPF map. The purpose is to * provide a simple user intuitive way to do * "cat bpffs/pathto/a-pinned-map". * * Other operations (e.g. write, lookup...) should be realized by * the userspace tools (e.g. bpftool) through the * BPF_OBJ_GET_INFO_BY_FD and the map's lookup/update * interface. */ static const struct file_operations bpffs_map_fops = { .open = bpffs_map_open, .read = seq_read, .release = bpffs_map_release, }; static int bpffs_obj_open(struct inode *inode, struct file *file) { return -EIO; } static const struct file_operations bpffs_obj_fops = { .open = bpffs_obj_open, }; static int bpf_mkobj_ops(struct dentry *dentry, umode_t mode, void *raw, const struct inode_operations *iops, const struct file_operations *fops) { struct inode *dir = dentry->d_parent->d_inode; struct inode *inode = bpf_get_inode(dir->i_sb, dir, mode); if (IS_ERR(inode)) return PTR_ERR(inode); inode->i_op = iops; inode->i_fop = fops; inode->i_private = raw; bpf_dentry_finalize(dentry, inode, dir); return 0; } static int bpf_mkprog(struct dentry *dentry, umode_t mode, void *arg) { return bpf_mkobj_ops(dentry, mode, arg, &bpf_prog_iops, &bpffs_obj_fops); } static int bpf_mkmap(struct dentry *dentry, umode_t mode, void *arg) { struct bpf_map *map = arg; return bpf_mkobj_ops(dentry, mode, arg, &bpf_map_iops, bpf_map_support_seq_show(map) ? &bpffs_map_fops : &bpffs_obj_fops); } static int bpf_mklink(struct dentry *dentry, umode_t mode, void *arg) { struct bpf_link *link = arg; return bpf_mkobj_ops(dentry, mode, arg, &bpf_link_iops, bpf_link_is_iter(link) ? &bpf_iter_fops : &bpffs_obj_fops); } static struct dentry * bpf_lookup(struct inode *dir, struct dentry *dentry, unsigned flags) { /* Dots in names (e.g. "/sys/fs/bpf/foo.bar") are reserved for future * extensions. That allows popoulate_bpffs() create special files. */ if ((dir->i_mode & S_IALLUGO) && strchr(dentry->d_name.name, '.')) return ERR_PTR(-EPERM); return simple_lookup(dir, dentry, flags); } static int bpf_symlink(struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry, const char *target) { char *link = kstrdup(target, GFP_USER | __GFP_NOWARN); struct inode *inode; if (!link) return -ENOMEM; inode = bpf_get_inode(dir->i_sb, dir, S_IRWXUGO | S_IFLNK); if (IS_ERR(inode)) { kfree(link); return PTR_ERR(inode); } inode->i_op = &simple_symlink_inode_operations; inode->i_link = link; bpf_dentry_finalize(dentry, inode, dir); return 0; } static const struct inode_operations bpf_dir_iops = { .lookup = bpf_lookup, .mkdir = bpf_mkdir, .symlink = bpf_symlink, .rmdir = simple_rmdir, .rename = simple_rename, .link = simple_link, .unlink = simple_unlink, }; /* pin iterator link into bpffs */ static int bpf_iter_link_pin_kernel(struct dentry *parent, const char *name, struct bpf_link *link) { umode_t mode = S_IFREG | S_IRUSR; struct dentry *dentry; int ret; inode_lock(parent->d_inode); dentry = lookup_one_len(name, parent, strlen(name)); if (IS_ERR(dentry)) { inode_unlock(parent->d_inode); return PTR_ERR(dentry); } ret = bpf_mkobj_ops(dentry, mode, link, &bpf_link_iops, &bpf_iter_fops); dput(dentry); inode_unlock(parent->d_inode); return ret; } static int bpf_obj_do_pin(int path_fd, const char __user *pathname, void *raw, enum bpf_type type) { struct dentry *dentry; struct inode *dir; struct path path; umode_t mode; int ret; dentry = user_path_create(path_fd, pathname, &path, 0); if (IS_ERR(dentry)) return PTR_ERR(dentry); dir = d_inode(path.dentry); if (dir->i_op != &bpf_dir_iops) { ret = -EPERM; goto out; } mode = S_IFREG | ((S_IRUSR | S_IWUSR) & ~current_umask()); ret = security_path_mknod(&path, dentry, mode, 0); if (ret) goto out; switch (type) { case BPF_TYPE_PROG: ret = vfs_mkobj(dentry, mode, bpf_mkprog, raw); break; case BPF_TYPE_MAP: ret = vfs_mkobj(dentry, mode, bpf_mkmap, raw); break; case BPF_TYPE_LINK: ret = vfs_mkobj(dentry, mode, bpf_mklink, raw); break; default: ret = -EPERM; } out: done_path_create(&path, dentry); return ret; } int bpf_obj_pin_user(u32 ufd, int path_fd, const char __user *pathname) { enum bpf_type type; void *raw; int ret; raw = bpf_fd_probe_obj(ufd, &type); if (IS_ERR(raw)) return PTR_ERR(raw); ret = bpf_obj_do_pin(path_fd, pathname, raw, type); if (ret != 0) bpf_any_put(raw, type); return ret; } static void *bpf_obj_do_get(int path_fd, const char __user *pathname, enum bpf_type *type, int flags) { struct inode *inode; struct path path; void *raw; int ret; ret = user_path_at(path_fd, pathname, LOOKUP_FOLLOW, &path); if (ret) return ERR_PTR(ret); inode = d_backing_inode(path.dentry); ret = path_permission(&path, ACC_MODE(flags)); if (ret) goto out; ret = bpf_inode_type(inode, type); if (ret) goto out; raw = bpf_any_get(inode->i_private, *type); if (!IS_ERR(raw)) touch_atime(&path); path_put(&path); return raw; out: path_put(&path); return ERR_PTR(ret); } int bpf_obj_get_user(int path_fd, const char __user *pathname, int flags) { enum bpf_type type = BPF_TYPE_UNSPEC; int f_flags; void *raw; int ret; f_flags = bpf_get_file_flag(flags); if (f_flags < 0) return f_flags; raw = bpf_obj_do_get(path_fd, pathname, &type, f_flags); if (IS_ERR(raw)) return PTR_ERR(raw); if (type == BPF_TYPE_PROG) ret = bpf_prog_new_fd(raw); else if (type == BPF_TYPE_MAP) ret = bpf_map_new_fd(raw, f_flags); else if (type == BPF_TYPE_LINK) ret = (f_flags != O_RDWR) ? -EINVAL : bpf_link_new_fd(raw); else return -ENOENT; if (ret < 0) bpf_any_put(raw, type); return ret; } static struct bpf_prog *__get_prog_inode(struct inode *inode, enum bpf_prog_type type) { struct bpf_prog *prog; int ret = inode_permission(&nop_mnt_idmap, inode, MAY_READ); if (ret) return ERR_PTR(ret); if (inode->i_op == &bpf_map_iops) return ERR_PTR(-EINVAL); if (inode->i_op == &bpf_link_iops) return ERR_PTR(-EINVAL); if (inode->i_op != &bpf_prog_iops) return ERR_PTR(-EACCES); prog = inode->i_private; ret = security_bpf_prog(prog); if (ret < 0) return ERR_PTR(ret); if (!bpf_prog_get_ok(prog, &type, false)) return ERR_PTR(-EINVAL); bpf_prog_inc(prog); return prog; } struct bpf_prog *bpf_prog_get_type_path(const char *name, enum bpf_prog_type type) { struct bpf_prog *prog; struct path path; int ret = kern_path(name, LOOKUP_FOLLOW, &path); if (ret) return ERR_PTR(ret); prog = __get_prog_inode(d_backing_inode(path.dentry), type); if (!IS_ERR(prog)) touch_atime(&path); path_put(&path); return prog; } EXPORT_SYMBOL(bpf_prog_get_type_path); struct bpffs_btf_enums { const struct btf *btf; const struct btf_type *cmd_t; const struct btf_type *map_t; const struct btf_type *prog_t; const struct btf_type *attach_t; }; static int find_bpffs_btf_enums(struct bpffs_btf_enums *info) { const struct btf *btf; const struct btf_type *t; const char *name; int i, n; memset(info, 0, sizeof(*info)); btf = bpf_get_btf_vmlinux(); if (IS_ERR(btf)) return PTR_ERR(btf); if (!btf) return -ENOENT; info->btf = btf; for (i = 1, n = btf_nr_types(btf); i < n; i++) { t = btf_type_by_id(btf, i); if (!btf_type_is_enum(t)) continue; name = btf_name_by_offset(btf, t->name_off); if (!name) continue; if (strcmp(name, "bpf_cmd") == 0) info->cmd_t = t; else if (strcmp(name, "bpf_map_type") == 0) info->map_t = t; else if (strcmp(name, "bpf_prog_type") == 0) info->prog_t = t; else if (strcmp(name, "bpf_attach_type") == 0) info->attach_t = t; else continue; if (info->cmd_t && info->map_t && info->prog_t && info->attach_t) return 0; } return -ESRCH; } static bool find_btf_enum_const(const struct btf *btf, const struct btf_type *enum_t, const char *prefix, const char *str, int *value) { const struct btf_enum *e; const char *name; int i, n, pfx_len = strlen(prefix); *value = 0; if (!btf || !enum_t) return false; for (i = 0, n = btf_vlen(enum_t); i < n; i++) { e = &btf_enum(enum_t)[i]; name = btf_name_by_offset(btf, e->name_off); if (!name || strncasecmp(name, prefix, pfx_len) != 0) continue; /* match symbolic name case insensitive and ignoring prefix */ if (strcasecmp(name + pfx_len, str) == 0) { *value = e->val; return true; } } return false; } static void seq_print_delegate_opts(struct seq_file *m, const char *opt_name, const struct btf *btf, const struct btf_type *enum_t, const char *prefix, u64 delegate_msk, u64 any_msk) { const struct btf_enum *e; bool first = true; const char *name; u64 msk; int i, n, pfx_len = strlen(prefix); delegate_msk &= any_msk; /* clear unknown bits */ if (delegate_msk == 0) return; seq_printf(m, ",%s", opt_name); if (delegate_msk == any_msk) { seq_printf(m, "=any"); return; } if (btf && enum_t) { for (i = 0, n = btf_vlen(enum_t); i < n; i++) { e = &btf_enum(enum_t)[i]; name = btf_name_by_offset(btf, e->name_off); if (!name || strncasecmp(name, prefix, pfx_len) != 0) continue; msk = 1ULL << e->val; if (delegate_msk & msk) { /* emit lower-case name without prefix */ seq_putc(m, first ? '=' : ':'); name += pfx_len; while (*name) { seq_putc(m, tolower(*name)); name++; } delegate_msk &= ~msk; first = false; } } } if (delegate_msk) seq_printf(m, "%c0x%llx", first ? '=' : ':', delegate_msk); } /* * Display the mount options in /proc/mounts. */ static int bpf_show_options(struct seq_file *m, struct dentry *root) { struct inode *inode = d_inode(root); umode_t mode = inode->i_mode & S_IALLUGO & ~S_ISVTX; struct bpf_mount_opts *opts = root->d_sb->s_fs_info; u64 mask; if (!uid_eq(inode->i_uid, GLOBAL_ROOT_UID)) seq_printf(m, ",uid=%u", from_kuid_munged(&init_user_ns, inode->i_uid)); if (!gid_eq(inode->i_gid, GLOBAL_ROOT_GID)) seq_printf(m, ",gid=%u", from_kgid_munged(&init_user_ns, inode->i_gid)); if (mode != S_IRWXUGO) seq_printf(m, ",mode=%o", mode); if (opts->delegate_cmds || opts->delegate_maps || opts->delegate_progs || opts->delegate_attachs) { struct bpffs_btf_enums info; /* ignore errors, fallback to hex */ (void)find_bpffs_btf_enums(&info); mask = (1ULL << __MAX_BPF_CMD) - 1; seq_print_delegate_opts(m, "delegate_cmds", info.btf, info.cmd_t, "BPF_", opts->delegate_cmds, mask); mask = (1ULL << __MAX_BPF_MAP_TYPE) - 1; seq_print_delegate_opts(m, "delegate_maps", info.btf, info.map_t, "BPF_MAP_TYPE_", opts->delegate_maps, mask); mask = (1ULL << __MAX_BPF_PROG_TYPE) - 1; seq_print_delegate_opts(m, "delegate_progs", info.btf, info.prog_t, "BPF_PROG_TYPE_", opts->delegate_progs, mask); mask = (1ULL << __MAX_BPF_ATTACH_TYPE) - 1; seq_print_delegate_opts(m, "delegate_attachs", info.btf, info.attach_t, "BPF_", opts->delegate_attachs, mask); } return 0; } static void bpf_free_inode(struct inode *inode) { enum bpf_type type; if (S_ISLNK(inode->i_mode)) kfree(inode->i_link); if (!bpf_inode_type(inode, &type)) bpf_any_put(inode->i_private, type); free_inode_nonrcu(inode); } const struct super_operations bpf_super_ops = { .statfs = simple_statfs, .drop_inode = generic_delete_inode, .show_options = bpf_show_options, .free_inode = bpf_free_inode, }; enum { OPT_UID, OPT_GID, OPT_MODE, OPT_DELEGATE_CMDS, OPT_DELEGATE_MAPS, OPT_DELEGATE_PROGS, OPT_DELEGATE_ATTACHS, }; static const struct fs_parameter_spec bpf_fs_parameters[] = { fsparam_u32 ("uid", OPT_UID), fsparam_u32 ("gid", OPT_GID), fsparam_u32oct ("mode", OPT_MODE), fsparam_string ("delegate_cmds", OPT_DELEGATE_CMDS), fsparam_string ("delegate_maps", OPT_DELEGATE_MAPS), fsparam_string ("delegate_progs", OPT_DELEGATE_PROGS), fsparam_string ("delegate_attachs", OPT_DELEGATE_ATTACHS), {} }; static int bpf_parse_param(struct fs_context *fc, struct fs_parameter *param) { struct bpf_mount_opts *opts = fc->s_fs_info; struct fs_parse_result result; kuid_t uid; kgid_t gid; int opt, err; opt = fs_parse(fc, bpf_fs_parameters, param, &result); if (opt < 0) { /* We might like to report bad mount options here, but * traditionally we've ignored all mount options, so we'd * better continue to ignore non-existing options for bpf. */ if (opt == -ENOPARAM) { opt = vfs_parse_fs_param_source(fc, param); if (opt != -ENOPARAM) return opt; return 0; } if (opt < 0) return opt; } switch (opt) { case OPT_UID: uid = make_kuid(current_user_ns(), result.uint_32); if (!uid_valid(uid)) goto bad_value; /* * The requested uid must be representable in the * filesystem's idmapping. */ if (!kuid_has_mapping(fc->user_ns, uid)) goto bad_value; opts->uid = uid; break; case OPT_GID: gid = make_kgid(current_user_ns(), result.uint_32); if (!gid_valid(gid)) goto bad_value; /* * The requested gid must be representable in the * filesystem's idmapping. */ if (!kgid_has_mapping(fc->user_ns, gid)) goto bad_value; opts->gid = gid; break; case OPT_MODE: opts->mode = result.uint_32 & S_IALLUGO; break; case OPT_DELEGATE_CMDS: case OPT_DELEGATE_MAPS: case OPT_DELEGATE_PROGS: case OPT_DELEGATE_ATTACHS: { struct bpffs_btf_enums info; const struct btf_type *enum_t; const char *enum_pfx; u64 *delegate_msk, msk = 0; char *p, *str; int val; /* ignore errors, fallback to hex */ (void)find_bpffs_btf_enums(&info); switch (opt) { case OPT_DELEGATE_CMDS: delegate_msk = &opts->delegate_cmds; enum_t = info.cmd_t; enum_pfx = "BPF_"; break; case OPT_DELEGATE_MAPS: delegate_msk = &opts->delegate_maps; enum_t = info.map_t; enum_pfx = "BPF_MAP_TYPE_"; break; case OPT_DELEGATE_PROGS: delegate_msk = &opts->delegate_progs; enum_t = info.prog_t; enum_pfx = "BPF_PROG_TYPE_"; break; case OPT_DELEGATE_ATTACHS: delegate_msk = &opts->delegate_attachs; enum_t = info.attach_t; enum_pfx = "BPF_"; break; default: return -EINVAL; } str = param->string; while ((p = strsep(&str, ":"))) { if (strcmp(p, "any") == 0) { msk |= ~0ULL; } else if (find_btf_enum_const(info.btf, enum_t, enum_pfx, p, &val)) { msk |= 1ULL << val; } else { err = kstrtou64(p, 0, &msk); if (err) return err; } } /* Setting delegation mount options requires privileges */ if (msk && !capable(CAP_SYS_ADMIN)) return -EPERM; *delegate_msk |= msk; break; } default: /* ignore unknown mount options */ break; } return 0; bad_value: return invalfc(fc, "Bad value for '%s'", param->key); } struct bpf_preload_ops *bpf_preload_ops; EXPORT_SYMBOL_GPL(bpf_preload_ops); static bool bpf_preload_mod_get(void) { /* If bpf_preload.ko wasn't loaded earlier then load it now. * When bpf_preload is built into vmlinux the module's __init * function will populate it. */ if (!bpf_preload_ops) { request_module("bpf_preload"); if (!bpf_preload_ops) return false; } /* And grab the reference, so the module doesn't disappear while the * kernel is interacting with the kernel module and its UMD. */ if (!try_module_get(bpf_preload_ops->owner)) { pr_err("bpf_preload module get failed.\n"); return false; } return true; } static void bpf_preload_mod_put(void) { if (bpf_preload_ops) /* now user can "rmmod bpf_preload" if necessary */ module_put(bpf_preload_ops->owner); } static DEFINE_MUTEX(bpf_preload_lock); static int populate_bpffs(struct dentry *parent) { struct bpf_preload_info objs[BPF_PRELOAD_LINKS] = {}; int err = 0, i; /* grab the mutex to make sure the kernel interactions with bpf_preload * are serialized */ mutex_lock(&bpf_preload_lock); /* if bpf_preload.ko wasn't built into vmlinux then load it */ if (!bpf_preload_mod_get()) goto out; err = bpf_preload_ops->preload(objs); if (err) goto out_put; for (i = 0; i < BPF_PRELOAD_LINKS; i++) { bpf_link_inc(objs[i].link); err = bpf_iter_link_pin_kernel(parent, objs[i].link_name, objs[i].link); if (err) { bpf_link_put(objs[i].link); goto out_put; } } out_put: bpf_preload_mod_put(); out: mutex_unlock(&bpf_preload_lock); return err; } static int bpf_fill_super(struct super_block *sb, struct fs_context *fc) { static const struct tree_descr bpf_rfiles[] = { { "" } }; struct bpf_mount_opts *opts = sb->s_fs_info; struct inode *inode; int ret; /* Mounting an instance of BPF FS requires privileges */ if (fc->user_ns != &init_user_ns && !capable(CAP_SYS_ADMIN)) return -EPERM; ret = simple_fill_super(sb, BPF_FS_MAGIC, bpf_rfiles); if (ret) return ret; sb->s_op = &bpf_super_ops; inode = sb->s_root->d_inode; inode->i_uid = opts->uid; inode->i_gid = opts->gid; inode->i_op = &bpf_dir_iops; inode->i_mode &= ~S_IALLUGO; populate_bpffs(sb->s_root); inode->i_mode |= S_ISVTX | opts->mode; return 0; } static int bpf_get_tree(struct fs_context *fc) { return get_tree_nodev(fc, bpf_fill_super); } static void bpf_free_fc(struct fs_context *fc) { kfree(fc->s_fs_info); } static const struct fs_context_operations bpf_context_ops = { .free = bpf_free_fc, .parse_param = bpf_parse_param, .get_tree = bpf_get_tree, }; /* * Set up the filesystem mount context. */ static int bpf_init_fs_context(struct fs_context *fc) { struct bpf_mount_opts *opts; opts = kzalloc(sizeof(struct bpf_mount_opts), GFP_KERNEL); if (!opts) return -ENOMEM; opts->mode = S_IRWXUGO; opts->uid = current_fsuid(); opts->gid = current_fsgid(); /* start out with no BPF token delegation enabled */ opts->delegate_cmds = 0; opts->delegate_maps = 0; opts->delegate_progs = 0; opts->delegate_attachs = 0; fc->s_fs_info = opts; fc->ops = &bpf_context_ops; return 0; } static void bpf_kill_super(struct super_block *sb) { struct bpf_mount_opts *opts = sb->s_fs_info; kill_litter_super(sb); kfree(opts); } static struct file_system_type bpf_fs_type = { .owner = THIS_MODULE, .name = "bpf", .init_fs_context = bpf_init_fs_context, .parameters = bpf_fs_parameters, .kill_sb = bpf_kill_super, .fs_flags = FS_USERNS_MOUNT, }; static int __init bpf_init(void) { int ret; ret = sysfs_create_mount_point(fs_kobj, "bpf"); if (ret) return ret; ret = register_filesystem(&bpf_fs_type); if (ret) sysfs_remove_mount_point(fs_kobj, "bpf"); return ret; } fs_initcall(bpf_init); |
| 301 291 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 | /* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2013 Red Hat, Inc. and Parallels Inc. All rights reserved. * Authors: David Chinner and Glauber Costa * * Generic LRU infrastructure */ #ifndef _LRU_LIST_H #define _LRU_LIST_H #include <linux/list.h> #include <linux/nodemask.h> #include <linux/shrinker.h> #include <linux/xarray.h> struct mem_cgroup; /* list_lru_walk_cb has to always return one of those */ enum lru_status { LRU_REMOVED, /* item removed from list */ LRU_REMOVED_RETRY, /* item removed, but lock has been dropped and reacquired */ LRU_ROTATE, /* item referenced, give another pass */ LRU_SKIP, /* item cannot be locked, skip */ LRU_RETRY, /* item not freeable. May drop the lock internally, but has to return locked. */ LRU_STOP, /* stop lru list walking. May drop the lock internally, but has to return locked. */ }; struct list_lru_one { struct list_head list; /* may become negative during memcg reparenting */ long nr_items; /* protects all fields above */ spinlock_t lock; }; struct list_lru_memcg { struct rcu_head rcu; /* array of per cgroup per node lists, indexed by node id */ struct list_lru_one node[]; }; struct list_lru_node { /* global list, used for the root cgroup in cgroup aware lrus */ struct list_lru_one lru; atomic_long_t nr_items; } ____cacheline_aligned_in_smp; struct list_lru { struct list_lru_node *node; #ifdef CONFIG_MEMCG struct list_head list; int shrinker_id; bool memcg_aware; struct xarray xa; #endif #ifdef CONFIG_LOCKDEP struct lock_class_key *key; #endif }; void list_lru_destroy(struct list_lru *lru); int __list_lru_init(struct list_lru *lru, bool memcg_aware, struct shrinker *shrinker); #define list_lru_init(lru) \ __list_lru_init((lru), false, NULL) #define list_lru_init_memcg(lru, shrinker) \ __list_lru_init((lru), true, shrinker) static inline int list_lru_init_memcg_key(struct list_lru *lru, struct shrinker *shrinker, struct lock_class_key *key) { #ifdef CONFIG_LOCKDEP lru->key = key; #endif return list_lru_init_memcg(lru, shrinker); } int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru, gfp_t gfp); void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *parent); /** * list_lru_add: add an element to the lru list's tail * @lru: the lru pointer * @item: the item to be added. * @nid: the node id of the sublist to add the item to. * @memcg: the cgroup of the sublist to add the item to. * * If the element is already part of a list, this function returns doing * nothing. Therefore the caller does not need to keep state about whether or * not the element already belongs in the list and is allowed to lazy update * it. Note however that this is valid for *a* list, not *this* list. If * the caller organize itself in a way that elements can be in more than * one type of list, it is up to the caller to fully remove the item from * the previous list (with list_lru_del() for instance) before moving it * to @lru. * * Return: true if the list was updated, false otherwise */ bool list_lru_add(struct list_lru *lru, struct list_head *item, int nid, struct mem_cgroup *memcg); /** * list_lru_add_obj: add an element to the lru list's tail * @lru: the lru pointer * @item: the item to be added. * * This function is similar to list_lru_add(), but the NUMA node and the * memcg of the sublist is determined by @item list_head. This assumption is * valid for slab objects LRU such as dentries, inodes, etc. * * Return value: true if the list was updated, false otherwise */ bool list_lru_add_obj(struct list_lru *lru, struct list_head *item); /** * list_lru_del: delete an element from the lru list * @lru: the lru pointer * @item: the item to be deleted. * @nid: the node id of the sublist to delete the item from. * @memcg: the cgroup of the sublist to delete the item from. * * This function works analogously as list_lru_add() in terms of list * manipulation. The comments about an element already pertaining to * a list are also valid for list_lru_del(). * * Return: true if the list was updated, false otherwise */ bool list_lru_del(struct list_lru *lru, struct list_head *item, int nid, struct mem_cgroup *memcg); /** * list_lru_del_obj: delete an element from the lru list * @lru: the lru pointer * @item: the item to be deleted. * * This function is similar to list_lru_del(), but the NUMA node and the * memcg of the sublist is determined by @item list_head. This assumption is * valid for slab objects LRU such as dentries, inodes, etc. * * Return value: true if the list was updated, false otherwise. */ bool list_lru_del_obj(struct list_lru *lru, struct list_head *item); /** * list_lru_count_one: return the number of objects currently held by @lru * @lru: the lru pointer. * @nid: the node id to count from. * @memcg: the cgroup to count from. * * There is no guarantee that the list is not updated while the count is being * computed. Callers that want such a guarantee need to provide an outer lock. * * Return: 0 for empty lists, otherwise the number of objects * currently held by @lru. */ unsigned long list_lru_count_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg); unsigned long list_lru_count_node(struct list_lru *lru, int nid); static inline unsigned long list_lru_shrink_count(struct list_lru *lru, struct shrink_control *sc) { return list_lru_count_one(lru, sc->nid, sc->memcg); } static inline unsigned long list_lru_count(struct list_lru *lru) { long count = 0; int nid; for_each_node_state(nid, N_NORMAL_MEMORY) count += list_lru_count_node(lru, nid); return count; } void list_lru_isolate(struct list_lru_one *list, struct list_head *item); void list_lru_isolate_move(struct list_lru_one *list, struct list_head *item, struct list_head *head); typedef enum lru_status (*list_lru_walk_cb)(struct list_head *item, struct list_lru_one *list, void *cb_arg); /** * list_lru_walk_one: walk a @lru, isolating and disposing freeable items. * @lru: the lru pointer. * @nid: the node id to scan from. * @memcg: the cgroup to scan from. * @isolate: callback function that is responsible for deciding what to do with * the item currently being scanned * @cb_arg: opaque type that will be passed to @isolate * @nr_to_walk: how many items to scan. * * This function will scan all elements in a particular @lru, calling the * @isolate callback for each of those items, along with the current list * spinlock and a caller-provided opaque. The @isolate callback can choose to * drop the lock internally, but *must* return with the lock held. The callback * will return an enum lru_status telling the @lru infrastructure what to * do with the object being scanned. * * Please note that @nr_to_walk does not mean how many objects will be freed, * just how many objects will be scanned. * * Return: the number of objects effectively removed from the LRU. */ unsigned long list_lru_walk_one(struct list_lru *lru, int nid, struct mem_cgroup *memcg, list_lru_walk_cb isolate, void *cb_arg, unsigned long *nr_to_walk); /** * list_lru_walk_one_irq: walk a @lru, isolating and disposing freeable items. * @lru: the lru pointer. * @nid: the node id to scan from. * @memcg: the cgroup to scan from. * @isolate: callback function that is responsible for deciding what to do with * the item currently being scanned * @cb_arg: opaque type that will be passed to @isolate * @nr_to_walk: how many items to scan. * * Same as list_lru_walk_one() except that the spinlock is acquired with * spin_lock_irq(). */ unsigned long list_lru_walk_one_irq(struct list_lru *lru, int nid, struct mem_cgroup *memcg, list_lru_walk_cb isolate, void *cb_arg, unsigned long *nr_to_walk); unsigned long list_lru_walk_node(struct list_lru *lru, int nid, list_lru_walk_cb isolate, void *cb_arg, unsigned long *nr_to_walk); static inline unsigned long list_lru_shrink_walk(struct list_lru *lru, struct shrink_control *sc, list_lru_walk_cb isolate, void *cb_arg) { return list_lru_walk_one(lru, sc->nid, sc->memcg, isolate, cb_arg, &sc->nr_to_scan); } static inline unsigned long list_lru_shrink_walk_irq(struct list_lru *lru, struct shrink_control *sc, list_lru_walk_cb isolate, void *cb_arg) { return list_lru_walk_one_irq(lru, sc->nid, sc->memcg, isolate, cb_arg, &sc->nr_to_scan); } static inline unsigned long list_lru_walk(struct list_lru *lru, list_lru_walk_cb isolate, void *cb_arg, unsigned long nr_to_walk) { long isolated = 0; int nid; for_each_node_state(nid, N_NORMAL_MEMORY) { isolated += list_lru_walk_node(lru, nid, isolate, cb_arg, &nr_to_walk); if (nr_to_walk <= 0) break; } return isolated; } #endif /* _LRU_LIST_H */ |
| 3 3 3 3 3 3 2 3 3 3 2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 | // SPDX-License-Identifier: GPL-2.0-or-later /* * CCM: Counter with CBC-MAC * * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> */ #include <crypto/internal/aead.h> #include <crypto/internal/cipher.h> #include <crypto/internal/hash.h> #include <crypto/internal/skcipher.h> #include <crypto/scatterwalk.h> #include <linux/err.h> #include <linux/init.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/slab.h> struct ccm_instance_ctx { struct crypto_skcipher_spawn ctr; struct crypto_ahash_spawn mac; }; struct crypto_ccm_ctx { struct crypto_ahash *mac; struct crypto_skcipher *ctr; }; struct crypto_rfc4309_ctx { struct crypto_aead *child; u8 nonce[3]; }; struct crypto_rfc4309_req_ctx { struct scatterlist src[3]; struct scatterlist dst[3]; struct aead_request subreq; }; struct crypto_ccm_req_priv_ctx { u8 odata[16]; u8 idata[16]; u8 auth_tag[16]; u32 flags; struct scatterlist src[3]; struct scatterlist dst[3]; union { struct ahash_request ahreq; struct skcipher_request skreq; }; }; struct cbcmac_tfm_ctx { struct crypto_cipher *child; }; struct cbcmac_desc_ctx { unsigned int len; u8 dg[]; }; static inline struct crypto_ccm_req_priv_ctx *crypto_ccm_reqctx( struct aead_request *req) { unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); } static int set_msg_len(u8 *block, unsigned int msglen, int csize) { __be32 data; memset(block, 0, csize); block += csize; if (csize >= 4) csize = 4; else if (msglen > (1 << (8 * csize))) return -EOVERFLOW; data = cpu_to_be32(msglen); memcpy(block - csize, (u8 *)&data + 4 - csize, csize); return 0; } static int crypto_ccm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_skcipher *ctr = ctx->ctr; struct crypto_ahash *mac = ctx->mac; int err; crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) & CRYPTO_TFM_REQ_MASK); err = crypto_skcipher_setkey(ctr, key, keylen); if (err) return err; crypto_ahash_clear_flags(mac, CRYPTO_TFM_REQ_MASK); crypto_ahash_set_flags(mac, crypto_aead_get_flags(aead) & CRYPTO_TFM_REQ_MASK); return crypto_ahash_setkey(mac, key, keylen); } static int crypto_ccm_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { switch (authsize) { case 4: case 6: case 8: case 10: case 12: case 14: case 16: break; default: return -EINVAL; } return 0; } static int format_input(u8 *info, struct aead_request *req, unsigned int cryptlen) { struct crypto_aead *aead = crypto_aead_reqtfm(req); unsigned int lp = req->iv[0]; unsigned int l = lp + 1; unsigned int m; m = crypto_aead_authsize(aead); memcpy(info, req->iv, 16); /* format control info per RFC 3610 and * NIST Special Publication 800-38C */ *info |= (8 * ((m - 2) / 2)); if (req->assoclen) *info |= 64; return set_msg_len(info + 16 - l, cryptlen, l); } static int format_adata(u8 *adata, unsigned int a) { int len = 0; /* add control info for associated data * RFC 3610 and NIST Special Publication 800-38C */ if (a < 65280) { *(__be16 *)adata = cpu_to_be16(a); len = 2; } else { *(__be16 *)adata = cpu_to_be16(0xfffe); *(__be32 *)&adata[2] = cpu_to_be32(a); len = 6; } return len; } static int crypto_ccm_auth(struct aead_request *req, struct scatterlist *plain, unsigned int cryptlen) { struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req); struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead); struct ahash_request *ahreq = &pctx->ahreq; unsigned int assoclen = req->assoclen; struct scatterlist sg[3]; u8 *odata = pctx->odata; u8 *idata = pctx->idata; int ilen, err; /* format control data for input */ err = format_input(odata, req, cryptlen); if (err) goto out; sg_init_table(sg, 3); sg_set_buf(&sg[0], odata, 16); /* format associated data and compute into mac */ if (assoclen) { ilen = format_adata(idata, assoclen); sg_set_buf(&sg[1], idata, ilen); sg_chain(sg, 3, req->src); } else { ilen = 0; sg_chain(sg, 2, req->src); } ahash_request_set_tfm(ahreq, ctx->mac); ahash_request_set_callback(ahreq, pctx->flags, NULL, NULL); ahash_request_set_crypt(ahreq, sg, NULL, assoclen + ilen + 16); err = crypto_ahash_init(ahreq); if (err) goto out; err = crypto_ahash_update(ahreq); if (err) goto out; /* we need to pad the MAC input to a round multiple of the block size */ ilen = 16 - (assoclen + ilen) % 16; if (ilen < 16) { memset(idata, 0, ilen); sg_init_table(sg, 2); sg_set_buf(&sg[0], idata, ilen); if (plain) sg_chain(sg, 2, plain); plain = sg; cryptlen += ilen; } ahash_request_set_crypt(ahreq, plain, odata, cryptlen); err = crypto_ahash_finup(ahreq); out: return err; } static void crypto_ccm_encrypt_done(void *data, int err) { struct aead_request *req = data; struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req); u8 *odata = pctx->odata; if (!err) scatterwalk_map_and_copy(odata, req->dst, req->assoclen + req->cryptlen, crypto_aead_authsize(aead), 1); aead_request_complete(req, err); } static inline int crypto_ccm_check_iv(const u8 *iv) { /* 2 <= L <= 8, so 1 <= L' <= 7. */ if (1 > iv[0] || iv[0] > 7) return -EINVAL; return 0; } static int crypto_ccm_init_crypt(struct aead_request *req, u8 *tag) { struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req); struct scatterlist *sg; u8 *iv = req->iv; int err; err = crypto_ccm_check_iv(iv); if (err) return err; pctx->flags = aead_request_flags(req); /* Note: rfc 3610 and NIST 800-38C require counter of * zero to encrypt auth tag. */ memset(iv + 15 - iv[0], 0, iv[0] + 1); sg_init_table(pctx->src, 3); sg_set_buf(pctx->src, tag, 16); sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen); if (sg != pctx->src + 1) sg_chain(pctx->src, 2, sg); if (req->src != req->dst) { sg_init_table(pctx->dst, 3); sg_set_buf(pctx->dst, tag, 16); sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen); if (sg != pctx->dst + 1) sg_chain(pctx->dst, 2, sg); } return 0; } static int crypto_ccm_encrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req); struct skcipher_request *skreq = &pctx->skreq; struct scatterlist *dst; unsigned int cryptlen = req->cryptlen; u8 *odata = pctx->odata; u8 *iv = req->iv; int err; err = crypto_ccm_init_crypt(req, odata); if (err) return err; err = crypto_ccm_auth(req, sg_next(pctx->src), cryptlen); if (err) return err; dst = pctx->src; if (req->src != req->dst) dst = pctx->dst; skcipher_request_set_tfm(skreq, ctx->ctr); skcipher_request_set_callback(skreq, pctx->flags, crypto_ccm_encrypt_done, req); skcipher_request_set_crypt(skreq, pctx->src, dst, cryptlen + 16, iv); err = crypto_skcipher_encrypt(skreq); if (err) return err; /* copy authtag to end of dst */ scatterwalk_map_and_copy(odata, sg_next(dst), cryptlen, crypto_aead_authsize(aead), 1); return err; } static void crypto_ccm_decrypt_done(void *data, int err) { struct aead_request *req = data; struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req); struct crypto_aead *aead = crypto_aead_reqtfm(req); unsigned int authsize = crypto_aead_authsize(aead); unsigned int cryptlen = req->cryptlen - authsize; struct scatterlist *dst; pctx->flags = 0; dst = sg_next(req->src == req->dst ? pctx->src : pctx->dst); if (!err) { err = crypto_ccm_auth(req, dst, cryptlen); if (!err && crypto_memneq(pctx->auth_tag, pctx->odata, authsize)) err = -EBADMSG; } aead_request_complete(req, err); } static int crypto_ccm_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_ccm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_ccm_req_priv_ctx *pctx = crypto_ccm_reqctx(req); struct skcipher_request *skreq = &pctx->skreq; struct scatterlist *dst; unsigned int authsize = crypto_aead_authsize(aead); unsigned int cryptlen = req->cryptlen; u8 *authtag = pctx->auth_tag; u8 *odata = pctx->odata; u8 *iv = pctx->idata; int err; cryptlen -= authsize; err = crypto_ccm_init_crypt(req, authtag); if (err) return err; scatterwalk_map_and_copy(authtag, sg_next(pctx->src), cryptlen, authsize, 0); dst = pctx->src; if (req->src != req->dst) dst = pctx->dst; memcpy(iv, req->iv, 16); skcipher_request_set_tfm(skreq, ctx->ctr); skcipher_request_set_callback(skreq, pctx->flags, crypto_ccm_decrypt_done, req); skcipher_request_set_crypt(skreq, pctx->src, dst, cryptlen + 16, iv); err = crypto_skcipher_decrypt(skreq); if (err) return err; err = crypto_ccm_auth(req, sg_next(dst), cryptlen); if (err) return err; /* verify */ if (crypto_memneq(authtag, odata, authsize)) return -EBADMSG; return err; } static int crypto_ccm_init_tfm(struct crypto_aead *tfm) { struct aead_instance *inst = aead_alg_instance(tfm); struct ccm_instance_ctx *ictx = aead_instance_ctx(inst); struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_ahash *mac; struct crypto_skcipher *ctr; unsigned long align; int err; mac = crypto_spawn_ahash(&ictx->mac); if (IS_ERR(mac)) return PTR_ERR(mac); ctr = crypto_spawn_skcipher(&ictx->ctr); err = PTR_ERR(ctr); if (IS_ERR(ctr)) goto err_free_mac; ctx->mac = mac; ctx->ctr = ctr; align = crypto_aead_alignmask(tfm); align &= ~(crypto_tfm_ctx_alignment() - 1); crypto_aead_set_reqsize( tfm, align + sizeof(struct crypto_ccm_req_priv_ctx) + max(crypto_ahash_reqsize(mac), crypto_skcipher_reqsize(ctr))); return 0; err_free_mac: crypto_free_ahash(mac); return err; } static void crypto_ccm_exit_tfm(struct crypto_aead *tfm) { struct crypto_ccm_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_ahash(ctx->mac); crypto_free_skcipher(ctx->ctr); } static void crypto_ccm_free(struct aead_instance *inst) { struct ccm_instance_ctx *ctx = aead_instance_ctx(inst); crypto_drop_ahash(&ctx->mac); crypto_drop_skcipher(&ctx->ctr); kfree(inst); } static int crypto_ccm_create_common(struct crypto_template *tmpl, struct rtattr **tb, const char *ctr_name, const char *mac_name) { struct skcipher_alg_common *ctr; u32 mask; struct aead_instance *inst; struct ccm_instance_ctx *ictx; struct hash_alg_common *mac; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); if (err) return err; inst = kzalloc(sizeof(*inst) + sizeof(*ictx), GFP_KERNEL); if (!inst) return -ENOMEM; ictx = aead_instance_ctx(inst); err = crypto_grab_ahash(&ictx->mac, aead_crypto_instance(inst), mac_name, 0, mask | CRYPTO_ALG_ASYNC); if (err) goto err_free_inst; mac = crypto_spawn_ahash_alg(&ictx->mac); err = -EINVAL; if (strncmp(mac->base.cra_name, "cbcmac(", 7) != 0 || mac->digestsize != 16) goto err_free_inst; err = crypto_grab_skcipher(&ictx->ctr, aead_crypto_instance(inst), ctr_name, 0, mask); if (err) goto err_free_inst; ctr = crypto_spawn_skcipher_alg_common(&ictx->ctr); /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */ err = -EINVAL; if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 || ctr->ivsize != 16 || ctr->base.cra_blocksize != 1) goto err_free_inst; /* ctr and cbcmac must use the same underlying block cipher. */ if (strcmp(ctr->base.cra_name + 4, mac->base.cra_name + 7) != 0) goto err_free_inst; err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "ccm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME) goto err_free_inst; if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "ccm_base(%s,%s)", ctr->base.cra_driver_name, mac->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_free_inst; inst->alg.base.cra_priority = (mac->base.cra_priority + ctr->base.cra_priority) / 2; inst->alg.base.cra_blocksize = 1; inst->alg.base.cra_alignmask = ctr->base.cra_alignmask; inst->alg.ivsize = 16; inst->alg.chunksize = ctr->chunksize; inst->alg.maxauthsize = 16; inst->alg.base.cra_ctxsize = sizeof(struct crypto_ccm_ctx); inst->alg.init = crypto_ccm_init_tfm; inst->alg.exit = crypto_ccm_exit_tfm; inst->alg.setkey = crypto_ccm_setkey; inst->alg.setauthsize = crypto_ccm_setauthsize; inst->alg.encrypt = crypto_ccm_encrypt; inst->alg.decrypt = crypto_ccm_decrypt; inst->free = crypto_ccm_free; err = aead_register_instance(tmpl, inst); if (err) { err_free_inst: crypto_ccm_free(inst); } return err; } static int crypto_ccm_create(struct crypto_template *tmpl, struct rtattr **tb) { const char *cipher_name; char ctr_name[CRYPTO_MAX_ALG_NAME]; char mac_name[CRYPTO_MAX_ALG_NAME]; cipher_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(cipher_name)) return PTR_ERR(cipher_name); if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >= CRYPTO_MAX_ALG_NAME) return -ENAMETOOLONG; if (snprintf(mac_name, CRYPTO_MAX_ALG_NAME, "cbcmac(%s)", cipher_name) >= CRYPTO_MAX_ALG_NAME) return -ENAMETOOLONG; return crypto_ccm_create_common(tmpl, tb, ctr_name, mac_name); } static int crypto_ccm_base_create(struct crypto_template *tmpl, struct rtattr **tb) { const char *ctr_name; const char *mac_name; ctr_name = crypto_attr_alg_name(tb[1]); if (IS_ERR(ctr_name)) return PTR_ERR(ctr_name); mac_name = crypto_attr_alg_name(tb[2]); if (IS_ERR(mac_name)) return PTR_ERR(mac_name); return crypto_ccm_create_common(tmpl, tb, ctr_name, mac_name); } static int crypto_rfc4309_setkey(struct crypto_aead *parent, const u8 *key, unsigned int keylen) { struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(parent); struct crypto_aead *child = ctx->child; if (keylen < 3) return -EINVAL; keylen -= 3; memcpy(ctx->nonce, key + keylen, 3); crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & CRYPTO_TFM_REQ_MASK); return crypto_aead_setkey(child, key, keylen); } static int crypto_rfc4309_setauthsize(struct crypto_aead *parent, unsigned int authsize) { struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(parent); switch (authsize) { case 8: case 12: case 16: break; default: return -EINVAL; } return crypto_aead_setauthsize(ctx->child, authsize); } static struct aead_request *crypto_rfc4309_crypt(struct aead_request *req) { struct crypto_rfc4309_req_ctx *rctx = aead_request_ctx(req); struct aead_request *subreq = &rctx->subreq; struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(aead); struct crypto_aead *child = ctx->child; struct scatterlist *sg; u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child), crypto_aead_alignmask(child) + 1); /* L' */ iv[0] = 3; memcpy(iv + 1, ctx->nonce, 3); memcpy(iv + 4, req->iv, 8); scatterwalk_map_and_copy(iv + 16, req->src, 0, req->assoclen - 8, 0); sg_init_table(rctx->src, 3); sg_set_buf(rctx->src, iv + 16, req->assoclen - 8); sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen); if (sg != rctx->src + 1) sg_chain(rctx->src, 2, sg); if (req->src != req->dst) { sg_init_table(rctx->dst, 3); sg_set_buf(rctx->dst, iv + 16, req->assoclen - 8); sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen); if (sg != rctx->dst + 1) sg_chain(rctx->dst, 2, sg); } aead_request_set_tfm(subreq, child); aead_request_set_callback(subreq, req->base.flags, req->base.complete, req->base.data); aead_request_set_crypt(subreq, rctx->src, req->src == req->dst ? rctx->src : rctx->dst, req->cryptlen, iv); aead_request_set_ad(subreq, req->assoclen - 8); return subreq; } static int crypto_rfc4309_encrypt(struct aead_request *req) { if (req->assoclen != 16 && req->assoclen != 20) return -EINVAL; req = crypto_rfc4309_crypt(req); return crypto_aead_encrypt(req); } static int crypto_rfc4309_decrypt(struct aead_request *req) { if (req->assoclen != 16 && req->assoclen != 20) return -EINVAL; req = crypto_rfc4309_crypt(req); return crypto_aead_decrypt(req); } static int crypto_rfc4309_init_tfm(struct crypto_aead *tfm) { struct aead_instance *inst = aead_alg_instance(tfm); struct crypto_aead_spawn *spawn = aead_instance_ctx(inst); struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_aead *aead; unsigned long align; aead = crypto_spawn_aead(spawn); if (IS_ERR(aead)) return PTR_ERR(aead); ctx->child = aead; align = crypto_aead_alignmask(aead); align &= ~(crypto_tfm_ctx_alignment() - 1); crypto_aead_set_reqsize( tfm, sizeof(struct crypto_rfc4309_req_ctx) + ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) + align + 32); return 0; } static void crypto_rfc4309_exit_tfm(struct crypto_aead *tfm) { struct crypto_rfc4309_ctx *ctx = crypto_aead_ctx(tfm); crypto_free_aead(ctx->child); } static void crypto_rfc4309_free(struct aead_instance *inst) { crypto_drop_aead(aead_instance_ctx(inst)); kfree(inst); } static int crypto_rfc4309_create(struct crypto_template *tmpl, struct rtattr **tb) { u32 mask; struct aead_instance *inst; struct crypto_aead_spawn *spawn; struct aead_alg *alg; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, &mask); if (err) return err; inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); if (!inst) return -ENOMEM; spawn = aead_instance_ctx(inst); err = crypto_grab_aead(spawn, aead_crypto_instance(inst), crypto_attr_alg_name(tb[1]), 0, mask); if (err) goto err_free_inst; alg = crypto_spawn_aead_alg(spawn); err = -EINVAL; /* We only support 16-byte blocks. */ if (crypto_aead_alg_ivsize(alg) != 16) goto err_free_inst; /* Not a stream cipher? */ if (alg->base.cra_blocksize != 1) goto err_free_inst; err = -ENAMETOOLONG; if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "rfc4309(%s)", alg->base.cra_name) >= CRYPTO_MAX_ALG_NAME || snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "rfc4309(%s)", alg->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_free_inst; inst->alg.base.cra_priority = alg->base.cra_priority; inst->alg.base.cra_blocksize = 1; inst->alg.base.cra_alignmask = alg->base.cra_alignmask; inst->alg.ivsize = 8; inst->alg.chunksize = crypto_aead_alg_chunksize(alg); inst->alg.maxauthsize = 16; inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4309_ctx); inst->alg.init = crypto_rfc4309_init_tfm; inst->alg.exit = crypto_rfc4309_exit_tfm; inst->alg.setkey = crypto_rfc4309_setkey; inst->alg.setauthsize = crypto_rfc4309_setauthsize; inst->alg.encrypt = crypto_rfc4309_encrypt; inst->alg.decrypt = crypto_rfc4309_decrypt; inst->free = crypto_rfc4309_free; err = aead_register_instance(tmpl, inst); if (err) { err_free_inst: crypto_rfc4309_free(inst); } return err; } static int crypto_cbcmac_digest_setkey(struct crypto_shash *parent, const u8 *inkey, unsigned int keylen) { struct cbcmac_tfm_ctx *ctx = crypto_shash_ctx(parent); return crypto_cipher_setkey(ctx->child, inkey, keylen); } static int crypto_cbcmac_digest_init(struct shash_desc *pdesc) { struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc); int bs = crypto_shash_digestsize(pdesc->tfm); ctx->len = 0; memset(ctx->dg, 0, bs); return 0; } static int crypto_cbcmac_digest_update(struct shash_desc *pdesc, const u8 *p, unsigned int len) { struct crypto_shash *parent = pdesc->tfm; struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent); struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_digestsize(parent); while (len > 0) { unsigned int l = min(len, bs - ctx->len); crypto_xor(&ctx->dg[ctx->len], p, l); ctx->len +=l; len -= l; p += l; if (ctx->len == bs) { crypto_cipher_encrypt_one(tfm, ctx->dg, ctx->dg); ctx->len = 0; } } return 0; } static int crypto_cbcmac_digest_final(struct shash_desc *pdesc, u8 *out) { struct crypto_shash *parent = pdesc->tfm; struct cbcmac_tfm_ctx *tctx = crypto_shash_ctx(parent); struct cbcmac_desc_ctx *ctx = shash_desc_ctx(pdesc); struct crypto_cipher *tfm = tctx->child; int bs = crypto_shash_digestsize(parent); if (ctx->len) crypto_cipher_encrypt_one(tfm, ctx->dg, ctx->dg); memcpy(out, ctx->dg, bs); return 0; } static int cbcmac_init_tfm(struct crypto_tfm *tfm) { struct crypto_cipher *cipher; struct crypto_instance *inst = (void *)tfm->__crt_alg; struct crypto_cipher_spawn *spawn = crypto_instance_ctx(inst); struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm); cipher = crypto_spawn_cipher(spawn); if (IS_ERR(cipher)) return PTR_ERR(cipher); ctx->child = cipher; return 0; }; static void cbcmac_exit_tfm(struct crypto_tfm *tfm) { struct cbcmac_tfm_ctx *ctx = crypto_tfm_ctx(tfm); crypto_free_cipher(ctx->child); } static int cbcmac_create(struct crypto_template *tmpl, struct rtattr **tb) { struct shash_instance *inst; struct crypto_cipher_spawn *spawn; struct crypto_alg *alg; u32 mask; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH, &mask); if (err) return err; inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); if (!inst) return -ENOMEM; spawn = shash_instance_ctx(inst); err = crypto_grab_cipher(spawn, shash_crypto_instance(inst), crypto_attr_alg_name(tb[1]), 0, mask); if (err) goto err_free_inst; alg = crypto_spawn_cipher_alg(spawn); err = crypto_inst_setname(shash_crypto_instance(inst), tmpl->name, alg); if (err) goto err_free_inst; inst->alg.base.cra_priority = alg->cra_priority; inst->alg.base.cra_blocksize = 1; inst->alg.digestsize = alg->cra_blocksize; inst->alg.descsize = sizeof(struct cbcmac_desc_ctx) + alg->cra_blocksize; inst->alg.base.cra_ctxsize = sizeof(struct cbcmac_tfm_ctx); inst->alg.base.cra_init = cbcmac_init_tfm; inst->alg.base.cra_exit = cbcmac_exit_tfm; inst->alg.init = crypto_cbcmac_digest_init; inst->alg.update = crypto_cbcmac_digest_update; inst->alg.final = crypto_cbcmac_digest_final; inst->alg.setkey = crypto_cbcmac_digest_setkey; inst->free = shash_free_singlespawn_instance; err = shash_register_instance(tmpl, inst); if (err) { err_free_inst: shash_free_singlespawn_instance(inst); } return err; } static struct crypto_template crypto_ccm_tmpls[] = { { .name = "cbcmac", .create = cbcmac_create, .module = THIS_MODULE, }, { .name = "ccm_base", .create = crypto_ccm_base_create, .module = THIS_MODULE, }, { .name = "ccm", .create = crypto_ccm_create, .module = THIS_MODULE, }, { .name = "rfc4309", .create = crypto_rfc4309_create, .module = THIS_MODULE, }, }; static int __init crypto_ccm_module_init(void) { return crypto_register_templates(crypto_ccm_tmpls, ARRAY_SIZE(crypto_ccm_tmpls)); } static void __exit crypto_ccm_module_exit(void) { crypto_unregister_templates(crypto_ccm_tmpls, ARRAY_SIZE(crypto_ccm_tmpls)); } subsys_initcall(crypto_ccm_module_init); module_exit(crypto_ccm_module_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Counter with CBC MAC"); MODULE_ALIAS_CRYPTO("ccm_base"); MODULE_ALIAS_CRYPTO("rfc4309"); MODULE_ALIAS_CRYPTO("ccm"); MODULE_ALIAS_CRYPTO("cbcmac"); MODULE_IMPORT_NS("CRYPTO_INTERNAL"); |
| 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 | // SPDX-License-Identifier: GPL-2.0-or-later /*************************************************************************** * * Copyright (C) 2007-2010 SMSC * *****************************************************************************/ #include <linux/module.h> #include <linux/kmod.h> #include <linux/netdevice.h> #include <linux/etherdevice.h> #include <linux/ethtool.h> #include <linux/mii.h> #include <linux/usb.h> #include <linux/bitrev.h> #include <linux/crc16.h> #include <linux/crc32.h> #include <linux/usb/usbnet.h> #include <linux/slab.h> #include <linux/of_net.h> #include "smsc75xx.h" #define SMSC_CHIPNAME "smsc75xx" #define SMSC_DRIVER_VERSION "1.0.0" #define HS_USB_PKT_SIZE (512) #define FS_USB_PKT_SIZE (64) #define DEFAULT_HS_BURST_CAP_SIZE (16 * 1024 + 5 * HS_USB_PKT_SIZE) #define DEFAULT_FS_BURST_CAP_SIZE (6 * 1024 + 33 * FS_USB_PKT_SIZE) #define DEFAULT_BULK_IN_DELAY (0x00002000) #define MAX_SINGLE_PACKET_SIZE (9000) #define LAN75XX_EEPROM_MAGIC (0x7500) #define EEPROM_MAC_OFFSET (0x01) #define DEFAULT_TX_CSUM_ENABLE (true) #define DEFAULT_RX_CSUM_ENABLE (true) #define SMSC75XX_INTERNAL_PHY_ID (1) #define SMSC75XX_TX_OVERHEAD (8) #define MAX_RX_FIFO_SIZE (20 * 1024) #define MAX_TX_FIFO_SIZE (12 * 1024) #define USB_VENDOR_ID_SMSC (0x0424) #define USB_PRODUCT_ID_LAN7500 (0x7500) #define USB_PRODUCT_ID_LAN7505 (0x7505) #define RXW_PADDING 2 #define SUPPORTED_WAKE (WAKE_PHY | WAKE_UCAST | WAKE_BCAST | \ WAKE_MCAST | WAKE_ARP | WAKE_MAGIC) #define SUSPEND_SUSPEND0 (0x01) #define SUSPEND_SUSPEND1 (0x02) #define SUSPEND_SUSPEND2 (0x04) #define SUSPEND_SUSPEND3 (0x08) #define SUSPEND_ALLMODES (SUSPEND_SUSPEND0 | SUSPEND_SUSPEND1 | \ SUSPEND_SUSPEND2 | SUSPEND_SUSPEND3) struct smsc75xx_priv { struct usbnet *dev; u32 rfe_ctl; u32 wolopts; u32 multicast_hash_table[DP_SEL_VHF_HASH_LEN]; struct mutex dataport_mutex; spinlock_t rfe_ctl_lock; struct work_struct set_multicast; u8 suspend_flags; }; static bool turbo_mode = true; module_param(turbo_mode, bool, 0644); MODULE_PARM_DESC(turbo_mode, "Enable multiple frames per Rx transaction"); static int smsc75xx_link_ok_nopm(struct usbnet *dev); static int smsc75xx_phy_gig_workaround(struct usbnet *dev); static int __must_check __smsc75xx_read_reg(struct usbnet *dev, u32 index, u32 *data, int in_pm) { u32 buf; int ret; int (*fn)(struct usbnet *, u8, u8, u16, u16, void *, u16); BUG_ON(!dev); if (!in_pm) fn = usbnet_read_cmd; else fn = usbnet_read_cmd_nopm; ret = fn(dev, USB_VENDOR_REQUEST_READ_REGISTER, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, 0, index, &buf, 4); if (unlikely(ret < 4)) { ret = ret < 0 ? ret : -ENODATA; netdev_warn(dev->net, "Failed to read reg index 0x%08x: %d\n", index, ret); return ret; } le32_to_cpus(&buf); *data = buf; return ret; } static int __must_check __smsc75xx_write_reg(struct usbnet *dev, u32 index, u32 data, int in_pm) { u32 buf; int ret; int (*fn)(struct usbnet *, u8, u8, u16, u16, const void *, u16); BUG_ON(!dev); if (!in_pm) fn = usbnet_write_cmd; else fn = usbnet_write_cmd_nopm; buf = data; cpu_to_le32s(&buf); ret = fn(dev, USB_VENDOR_REQUEST_WRITE_REGISTER, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE, 0, index, &buf, 4); if (unlikely(ret < 0)) netdev_warn(dev->net, "Failed to write reg index 0x%08x: %d\n", index, ret); return ret; } static int __must_check smsc75xx_read_reg_nopm(struct usbnet *dev, u32 index, u32 *data) { return __smsc75xx_read_reg(dev, index, data, 1); } static int __must_check smsc75xx_write_reg_nopm(struct usbnet *dev, u32 index, u32 data) { return __smsc75xx_write_reg(dev, index, data, 1); } static int __must_check smsc75xx_read_reg(struct usbnet *dev, u32 index, u32 *data) { return __smsc75xx_read_reg(dev, index, data, 0); } static int __must_check smsc75xx_write_reg(struct usbnet *dev, u32 index, u32 data) { return __smsc75xx_write_reg(dev, index, data, 0); } /* Loop until the read is completed with timeout * called with phy_mutex held */ static __must_check int __smsc75xx_phy_wait_not_busy(struct usbnet *dev, int in_pm) { unsigned long start_time = jiffies; u32 val; int ret; do { ret = __smsc75xx_read_reg(dev, MII_ACCESS, &val, in_pm); if (ret < 0) { netdev_warn(dev->net, "Error reading MII_ACCESS\n"); return ret; } if (!(val & MII_ACCESS_BUSY)) return 0; } while (!time_after(jiffies, start_time + HZ)); return -EIO; } static int __smsc75xx_mdio_read(struct net_device *netdev, int phy_id, int idx, int in_pm) { struct usbnet *dev = netdev_priv(netdev); u32 val, addr; int ret; mutex_lock(&dev->phy_mutex); /* confirm MII not busy */ ret = __smsc75xx_phy_wait_not_busy(dev, in_pm); if (ret < 0) { netdev_warn(dev->net, "MII is busy in smsc75xx_mdio_read\n"); goto done; } /* set the address, index & direction (read from PHY) */ phy_id &= dev->mii.phy_id_mask; idx &= dev->mii.reg_num_mask; addr = ((phy_id << MII_ACCESS_PHY_ADDR_SHIFT) & MII_ACCESS_PHY_ADDR) | ((idx << MII_ACCESS_REG_ADDR_SHIFT) & MII_ACCESS_REG_ADDR) | MII_ACCESS_READ | MII_ACCESS_BUSY; ret = __smsc75xx_write_reg(dev, MII_ACCESS, addr, in_pm); if (ret < 0) { netdev_warn(dev->net, "Error writing MII_ACCESS\n"); goto done; } ret = __smsc75xx_phy_wait_not_busy(dev, in_pm); if (ret < 0) { netdev_warn(dev->net, "Timed out reading MII reg %02X\n", idx); goto done; } ret = __smsc75xx_read_reg(dev, MII_DATA, &val, in_pm); if (ret < 0) { netdev_warn(dev->net, "Error reading MII_DATA\n"); goto done; } ret = (u16)(val & 0xFFFF); done: mutex_unlock(&dev->phy_mutex); return ret; } static void __smsc75xx_mdio_write(struct net_device *netdev, int phy_id, int idx, int regval, int in_pm) { struct usbnet *dev = netdev_priv(netdev); u32 val, addr; int ret; mutex_lock(&dev->phy_mutex); /* confirm MII not busy */ ret = __smsc75xx_phy_wait_not_busy(dev, in_pm); if (ret < 0) { netdev_warn(dev->net, "MII is busy in smsc75xx_mdio_write\n"); goto done; } val = regval; ret = __smsc75xx_write_reg(dev, MII_DATA, val, in_pm); if (ret < 0) { netdev_warn(dev->net, "Error writing MII_DATA\n"); goto done; } /* set the address, index & direction (write to PHY) */ phy_id &= dev->mii.phy_id_mask; idx &= dev->mii.reg_num_mask; addr = ((phy_id << MII_ACCESS_PHY_ADDR_SHIFT) & MII_ACCESS_PHY_ADDR) | ((idx << MII_ACCESS_REG_ADDR_SHIFT) & MII_ACCESS_REG_ADDR) | MII_ACCESS_WRITE | MII_ACCESS_BUSY; ret = __smsc75xx_write_reg(dev, MII_ACCESS, addr, in_pm); if (ret < 0) { netdev_warn(dev->net, "Error writing MII_ACCESS\n"); goto done; } ret = __smsc75xx_phy_wait_not_busy(dev, in_pm); if (ret < 0) { netdev_warn(dev->net, "Timed out writing MII reg %02X\n", idx); goto done; } done: mutex_unlock(&dev->phy_mutex); } static int smsc75xx_mdio_read_nopm(struct net_device *netdev, int phy_id, int idx) { return __smsc75xx_mdio_read(netdev, phy_id, idx, 1); } static void smsc75xx_mdio_write_nopm(struct net_device *netdev, int phy_id, int idx, int regval) { __smsc75xx_mdio_write(netdev, phy_id, idx, regval, 1); } static int smsc75xx_mdio_read(struct net_device *netdev, int phy_id, int idx) { return __smsc75xx_mdio_read(netdev, phy_id, idx, 0); } static void smsc75xx_mdio_write(struct net_device *netdev, int phy_id, int idx, int regval) { __smsc75xx_mdio_write(netdev, phy_id, idx, regval, 0); } static int smsc75xx_wait_eeprom(struct usbnet *dev) { unsigned long start_time = jiffies; u32 val; int ret; do { ret = smsc75xx_read_reg(dev, E2P_CMD, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading E2P_CMD\n"); return ret; } if (!(val & E2P_CMD_BUSY) || (val & E2P_CMD_TIMEOUT)) break; udelay(40); } while (!time_after(jiffies, start_time + HZ)); if (val & (E2P_CMD_TIMEOUT | E2P_CMD_BUSY)) { netdev_warn(dev->net, "EEPROM read operation timeout\n"); return -EIO; } return 0; } static int smsc75xx_eeprom_confirm_not_busy(struct usbnet *dev) { unsigned long start_time = jiffies; u32 val; int ret; do { ret = smsc75xx_read_reg(dev, E2P_CMD, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading E2P_CMD\n"); return ret; } if (!(val & E2P_CMD_BUSY)) return 0; udelay(40); } while (!time_after(jiffies, start_time + HZ)); netdev_warn(dev->net, "EEPROM is busy\n"); return -EIO; } static int smsc75xx_read_eeprom(struct usbnet *dev, u32 offset, u32 length, u8 *data) { u32 val; int i, ret; BUG_ON(!dev); BUG_ON(!data); ret = smsc75xx_eeprom_confirm_not_busy(dev); if (ret) return ret; for (i = 0; i < length; i++) { val = E2P_CMD_BUSY | E2P_CMD_READ | (offset & E2P_CMD_ADDR); ret = smsc75xx_write_reg(dev, E2P_CMD, val); if (ret < 0) { netdev_warn(dev->net, "Error writing E2P_CMD\n"); return ret; } ret = smsc75xx_wait_eeprom(dev); if (ret < 0) return ret; ret = smsc75xx_read_reg(dev, E2P_DATA, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading E2P_DATA\n"); return ret; } data[i] = val & 0xFF; offset++; } return 0; } static int smsc75xx_write_eeprom(struct usbnet *dev, u32 offset, u32 length, u8 *data) { u32 val; int i, ret; BUG_ON(!dev); BUG_ON(!data); ret = smsc75xx_eeprom_confirm_not_busy(dev); if (ret) return ret; /* Issue write/erase enable command */ val = E2P_CMD_BUSY | E2P_CMD_EWEN; ret = smsc75xx_write_reg(dev, E2P_CMD, val); if (ret < 0) { netdev_warn(dev->net, "Error writing E2P_CMD\n"); return ret; } ret = smsc75xx_wait_eeprom(dev); if (ret < 0) return ret; for (i = 0; i < length; i++) { /* Fill data register */ val = data[i]; ret = smsc75xx_write_reg(dev, E2P_DATA, val); if (ret < 0) { netdev_warn(dev->net, "Error writing E2P_DATA\n"); return ret; } /* Send "write" command */ val = E2P_CMD_BUSY | E2P_CMD_WRITE | (offset & E2P_CMD_ADDR); ret = smsc75xx_write_reg(dev, E2P_CMD, val); if (ret < 0) { netdev_warn(dev->net, "Error writing E2P_CMD\n"); return ret; } ret = smsc75xx_wait_eeprom(dev); if (ret < 0) return ret; offset++; } return 0; } static int smsc75xx_dataport_wait_not_busy(struct usbnet *dev) { int i, ret; for (i = 0; i < 100; i++) { u32 dp_sel; ret = smsc75xx_read_reg(dev, DP_SEL, &dp_sel); if (ret < 0) { netdev_warn(dev->net, "Error reading DP_SEL\n"); return ret; } if (dp_sel & DP_SEL_DPRDY) return 0; udelay(40); } netdev_warn(dev->net, "smsc75xx_dataport_wait_not_busy timed out\n"); return -EIO; } static int smsc75xx_dataport_write(struct usbnet *dev, u32 ram_select, u32 addr, u32 length, u32 *buf) { struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); u32 dp_sel; int i, ret; mutex_lock(&pdata->dataport_mutex); ret = smsc75xx_dataport_wait_not_busy(dev); if (ret < 0) { netdev_warn(dev->net, "smsc75xx_dataport_write busy on entry\n"); goto done; } ret = smsc75xx_read_reg(dev, DP_SEL, &dp_sel); if (ret < 0) { netdev_warn(dev->net, "Error reading DP_SEL\n"); goto done; } dp_sel &= ~DP_SEL_RSEL; dp_sel |= ram_select; ret = smsc75xx_write_reg(dev, DP_SEL, dp_sel); if (ret < 0) { netdev_warn(dev->net, "Error writing DP_SEL\n"); goto done; } for (i = 0; i < length; i++) { ret = smsc75xx_write_reg(dev, DP_ADDR, addr + i); if (ret < 0) { netdev_warn(dev->net, "Error writing DP_ADDR\n"); goto done; } ret = smsc75xx_write_reg(dev, DP_DATA, buf[i]); if (ret < 0) { netdev_warn(dev->net, "Error writing DP_DATA\n"); goto done; } ret = smsc75xx_write_reg(dev, DP_CMD, DP_CMD_WRITE); if (ret < 0) { netdev_warn(dev->net, "Error writing DP_CMD\n"); goto done; } ret = smsc75xx_dataport_wait_not_busy(dev); if (ret < 0) { netdev_warn(dev->net, "smsc75xx_dataport_write timeout\n"); goto done; } } done: mutex_unlock(&pdata->dataport_mutex); return ret; } /* returns hash bit number for given MAC address */ static u32 smsc75xx_hash(char addr[ETH_ALEN]) { return (ether_crc(ETH_ALEN, addr) >> 23) & 0x1ff; } static void smsc75xx_deferred_multicast_write(struct work_struct *param) { struct smsc75xx_priv *pdata = container_of(param, struct smsc75xx_priv, set_multicast); struct usbnet *dev = pdata->dev; int ret; netif_dbg(dev, drv, dev->net, "deferred multicast write 0x%08x\n", pdata->rfe_ctl); smsc75xx_dataport_write(dev, DP_SEL_VHF, DP_SEL_VHF_VLAN_LEN, DP_SEL_VHF_HASH_LEN, pdata->multicast_hash_table); ret = smsc75xx_write_reg(dev, RFE_CTL, pdata->rfe_ctl); if (ret < 0) netdev_warn(dev->net, "Error writing RFE_CRL\n"); } static void smsc75xx_set_multicast(struct net_device *netdev) { struct usbnet *dev = netdev_priv(netdev); struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); unsigned long flags; int i; spin_lock_irqsave(&pdata->rfe_ctl_lock, flags); pdata->rfe_ctl &= ~(RFE_CTL_AU | RFE_CTL_AM | RFE_CTL_DPF | RFE_CTL_MHF); pdata->rfe_ctl |= RFE_CTL_AB; for (i = 0; i < DP_SEL_VHF_HASH_LEN; i++) pdata->multicast_hash_table[i] = 0; if (dev->net->flags & IFF_PROMISC) { netif_dbg(dev, drv, dev->net, "promiscuous mode enabled\n"); pdata->rfe_ctl |= RFE_CTL_AM | RFE_CTL_AU; } else if (dev->net->flags & IFF_ALLMULTI) { netif_dbg(dev, drv, dev->net, "receive all multicast enabled\n"); pdata->rfe_ctl |= RFE_CTL_AM | RFE_CTL_DPF; } else if (!netdev_mc_empty(dev->net)) { struct netdev_hw_addr *ha; netif_dbg(dev, drv, dev->net, "receive multicast hash filter\n"); pdata->rfe_ctl |= RFE_CTL_MHF | RFE_CTL_DPF; netdev_for_each_mc_addr(ha, netdev) { u32 bitnum = smsc75xx_hash(ha->addr); pdata->multicast_hash_table[bitnum / 32] |= (1 << (bitnum % 32)); } } else { netif_dbg(dev, drv, dev->net, "receive own packets only\n"); pdata->rfe_ctl |= RFE_CTL_DPF; } spin_unlock_irqrestore(&pdata->rfe_ctl_lock, flags); /* defer register writes to a sleepable context */ schedule_work(&pdata->set_multicast); } static int smsc75xx_update_flowcontrol(struct usbnet *dev, u8 duplex, u16 lcladv, u16 rmtadv) { u32 flow = 0, fct_flow = 0; int ret; if (duplex == DUPLEX_FULL) { u8 cap = mii_resolve_flowctrl_fdx(lcladv, rmtadv); if (cap & FLOW_CTRL_TX) { flow = (FLOW_TX_FCEN | 0xFFFF); /* set fct_flow thresholds to 20% and 80% */ fct_flow = (8 << 8) | 32; } if (cap & FLOW_CTRL_RX) flow |= FLOW_RX_FCEN; netif_dbg(dev, link, dev->net, "rx pause %s, tx pause %s\n", (cap & FLOW_CTRL_RX ? "enabled" : "disabled"), (cap & FLOW_CTRL_TX ? "enabled" : "disabled")); } else { netif_dbg(dev, link, dev->net, "half duplex\n"); } ret = smsc75xx_write_reg(dev, FLOW, flow); if (ret < 0) { netdev_warn(dev->net, "Error writing FLOW\n"); return ret; } ret = smsc75xx_write_reg(dev, FCT_FLOW, fct_flow); if (ret < 0) { netdev_warn(dev->net, "Error writing FCT_FLOW\n"); return ret; } return 0; } static int smsc75xx_link_reset(struct usbnet *dev) { struct mii_if_info *mii = &dev->mii; struct ethtool_cmd ecmd = { .cmd = ETHTOOL_GSET }; u16 lcladv, rmtadv; int ret; /* write to clear phy interrupt status */ smsc75xx_mdio_write(dev->net, mii->phy_id, PHY_INT_SRC, PHY_INT_SRC_CLEAR_ALL); ret = smsc75xx_write_reg(dev, INT_STS, INT_STS_CLEAR_ALL); if (ret < 0) { netdev_warn(dev->net, "Error writing INT_STS\n"); return ret; } mii_check_media(mii, 1, 1); mii_ethtool_gset(&dev->mii, &ecmd); lcladv = smsc75xx_mdio_read(dev->net, mii->phy_id, MII_ADVERTISE); rmtadv = smsc75xx_mdio_read(dev->net, mii->phy_id, MII_LPA); netif_dbg(dev, link, dev->net, "speed: %u duplex: %d lcladv: %04x rmtadv: %04x\n", ethtool_cmd_speed(&ecmd), ecmd.duplex, lcladv, rmtadv); return smsc75xx_update_flowcontrol(dev, ecmd.duplex, lcladv, rmtadv); } static void smsc75xx_status(struct usbnet *dev, struct urb *urb) { u32 intdata; if (urb->actual_length != 4) { netdev_warn(dev->net, "unexpected urb length %d\n", urb->actual_length); return; } intdata = get_unaligned_le32(urb->transfer_buffer); netif_dbg(dev, link, dev->net, "intdata: 0x%08X\n", intdata); if (intdata & INT_ENP_PHY_INT) usbnet_defer_kevent(dev, EVENT_LINK_RESET); else netdev_warn(dev->net, "unexpected interrupt, intdata=0x%08X\n", intdata); } static int smsc75xx_ethtool_get_eeprom_len(struct net_device *net) { return MAX_EEPROM_SIZE; } static int smsc75xx_ethtool_get_eeprom(struct net_device *netdev, struct ethtool_eeprom *ee, u8 *data) { struct usbnet *dev = netdev_priv(netdev); ee->magic = LAN75XX_EEPROM_MAGIC; return smsc75xx_read_eeprom(dev, ee->offset, ee->len, data); } static int smsc75xx_ethtool_set_eeprom(struct net_device *netdev, struct ethtool_eeprom *ee, u8 *data) { struct usbnet *dev = netdev_priv(netdev); if (ee->magic != LAN75XX_EEPROM_MAGIC) { netdev_warn(dev->net, "EEPROM: magic value mismatch: 0x%x\n", ee->magic); return -EINVAL; } return smsc75xx_write_eeprom(dev, ee->offset, ee->len, data); } static void smsc75xx_ethtool_get_wol(struct net_device *net, struct ethtool_wolinfo *wolinfo) { struct usbnet *dev = netdev_priv(net); struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); wolinfo->supported = SUPPORTED_WAKE; wolinfo->wolopts = pdata->wolopts; } static int smsc75xx_ethtool_set_wol(struct net_device *net, struct ethtool_wolinfo *wolinfo) { struct usbnet *dev = netdev_priv(net); struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); int ret; if (wolinfo->wolopts & ~SUPPORTED_WAKE) return -EINVAL; pdata->wolopts = wolinfo->wolopts & SUPPORTED_WAKE; ret = device_set_wakeup_enable(&dev->udev->dev, pdata->wolopts); if (ret < 0) netdev_warn(dev->net, "device_set_wakeup_enable error %d\n", ret); return ret; } static const struct ethtool_ops smsc75xx_ethtool_ops = { .get_link = usbnet_get_link, .nway_reset = usbnet_nway_reset, .get_drvinfo = usbnet_get_drvinfo, .get_msglevel = usbnet_get_msglevel, .set_msglevel = usbnet_set_msglevel, .get_eeprom_len = smsc75xx_ethtool_get_eeprom_len, .get_eeprom = smsc75xx_ethtool_get_eeprom, .set_eeprom = smsc75xx_ethtool_set_eeprom, .get_wol = smsc75xx_ethtool_get_wol, .set_wol = smsc75xx_ethtool_set_wol, .get_link_ksettings = usbnet_get_link_ksettings_mii, .set_link_ksettings = usbnet_set_link_ksettings_mii, }; static int smsc75xx_ioctl(struct net_device *netdev, struct ifreq *rq, int cmd) { struct usbnet *dev = netdev_priv(netdev); if (!netif_running(netdev)) return -EINVAL; return generic_mii_ioctl(&dev->mii, if_mii(rq), cmd, NULL); } static void smsc75xx_init_mac_address(struct usbnet *dev) { u8 addr[ETH_ALEN]; /* maybe the boot loader passed the MAC address in devicetree */ if (!platform_get_ethdev_address(&dev->udev->dev, dev->net)) { if (is_valid_ether_addr(dev->net->dev_addr)) { /* device tree values are valid so use them */ netif_dbg(dev, ifup, dev->net, "MAC address read from the device tree\n"); return; } } /* try reading mac address from EEPROM */ if (smsc75xx_read_eeprom(dev, EEPROM_MAC_OFFSET, ETH_ALEN, addr) == 0) { eth_hw_addr_set(dev->net, addr); if (is_valid_ether_addr(dev->net->dev_addr)) { /* eeprom values are valid so use them */ netif_dbg(dev, ifup, dev->net, "MAC address read from EEPROM\n"); return; } } /* no useful static MAC address found. generate a random one */ eth_hw_addr_random(dev->net); netif_dbg(dev, ifup, dev->net, "MAC address set to eth_random_addr\n"); } static int smsc75xx_set_mac_address(struct usbnet *dev) { u32 addr_lo = dev->net->dev_addr[0] | dev->net->dev_addr[1] << 8 | dev->net->dev_addr[2] << 16 | dev->net->dev_addr[3] << 24; u32 addr_hi = dev->net->dev_addr[4] | dev->net->dev_addr[5] << 8; int ret = smsc75xx_write_reg(dev, RX_ADDRH, addr_hi); if (ret < 0) { netdev_warn(dev->net, "Failed to write RX_ADDRH: %d\n", ret); return ret; } ret = smsc75xx_write_reg(dev, RX_ADDRL, addr_lo); if (ret < 0) { netdev_warn(dev->net, "Failed to write RX_ADDRL: %d\n", ret); return ret; } addr_hi |= ADDR_FILTX_FB_VALID; ret = smsc75xx_write_reg(dev, ADDR_FILTX, addr_hi); if (ret < 0) { netdev_warn(dev->net, "Failed to write ADDR_FILTX: %d\n", ret); return ret; } ret = smsc75xx_write_reg(dev, ADDR_FILTX + 4, addr_lo); if (ret < 0) netdev_warn(dev->net, "Failed to write ADDR_FILTX+4: %d\n", ret); return ret; } static int smsc75xx_phy_initialize(struct usbnet *dev) { int bmcr, ret, timeout = 0; /* Initialize MII structure */ dev->mii.dev = dev->net; dev->mii.mdio_read = smsc75xx_mdio_read; dev->mii.mdio_write = smsc75xx_mdio_write; dev->mii.phy_id_mask = 0x1f; dev->mii.reg_num_mask = 0x1f; dev->mii.supports_gmii = 1; dev->mii.phy_id = SMSC75XX_INTERNAL_PHY_ID; /* reset phy and wait for reset to complete */ smsc75xx_mdio_write(dev->net, dev->mii.phy_id, MII_BMCR, BMCR_RESET); do { msleep(10); bmcr = smsc75xx_mdio_read(dev->net, dev->mii.phy_id, MII_BMCR); if (bmcr < 0) { netdev_warn(dev->net, "Error reading MII_BMCR\n"); return bmcr; } timeout++; } while ((bmcr & BMCR_RESET) && (timeout < 100)); if (timeout >= 100) { netdev_warn(dev->net, "timeout on PHY Reset\n"); return -EIO; } /* phy workaround for gig link */ smsc75xx_phy_gig_workaround(dev); smsc75xx_mdio_write(dev->net, dev->mii.phy_id, MII_ADVERTISE, ADVERTISE_ALL | ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP | ADVERTISE_PAUSE_ASYM); smsc75xx_mdio_write(dev->net, dev->mii.phy_id, MII_CTRL1000, ADVERTISE_1000FULL); /* read and write to clear phy interrupt status */ ret = smsc75xx_mdio_read(dev->net, dev->mii.phy_id, PHY_INT_SRC); if (ret < 0) { netdev_warn(dev->net, "Error reading PHY_INT_SRC\n"); return ret; } smsc75xx_mdio_write(dev->net, dev->mii.phy_id, PHY_INT_SRC, 0xffff); smsc75xx_mdio_write(dev->net, dev->mii.phy_id, PHY_INT_MASK, PHY_INT_MASK_DEFAULT); mii_nway_restart(&dev->mii); netif_dbg(dev, ifup, dev->net, "phy initialised successfully\n"); return 0; } static int smsc75xx_set_rx_max_frame_length(struct usbnet *dev, int size) { int ret = 0; u32 buf; bool rxenabled; ret = smsc75xx_read_reg(dev, MAC_RX, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read MAC_RX: %d\n", ret); return ret; } rxenabled = ((buf & MAC_RX_RXEN) != 0); if (rxenabled) { buf &= ~MAC_RX_RXEN; ret = smsc75xx_write_reg(dev, MAC_RX, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write MAC_RX: %d\n", ret); return ret; } } /* add 4 to size for FCS */ buf &= ~MAC_RX_MAX_SIZE; buf |= (((size + 4) << MAC_RX_MAX_SIZE_SHIFT) & MAC_RX_MAX_SIZE); ret = smsc75xx_write_reg(dev, MAC_RX, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write MAC_RX: %d\n", ret); return ret; } if (rxenabled) { buf |= MAC_RX_RXEN; ret = smsc75xx_write_reg(dev, MAC_RX, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write MAC_RX: %d\n", ret); return ret; } } return 0; } static int smsc75xx_change_mtu(struct net_device *netdev, int new_mtu) { struct usbnet *dev = netdev_priv(netdev); int ret; ret = smsc75xx_set_rx_max_frame_length(dev, new_mtu + ETH_HLEN); if (ret < 0) { netdev_warn(dev->net, "Failed to set mac rx frame length\n"); return ret; } return usbnet_change_mtu(netdev, new_mtu); } /* Enable or disable Rx checksum offload engine */ static int smsc75xx_set_features(struct net_device *netdev, netdev_features_t features) { struct usbnet *dev = netdev_priv(netdev); struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); unsigned long flags; int ret; spin_lock_irqsave(&pdata->rfe_ctl_lock, flags); if (features & NETIF_F_RXCSUM) pdata->rfe_ctl |= RFE_CTL_TCPUDP_CKM | RFE_CTL_IP_CKM; else pdata->rfe_ctl &= ~(RFE_CTL_TCPUDP_CKM | RFE_CTL_IP_CKM); spin_unlock_irqrestore(&pdata->rfe_ctl_lock, flags); /* it's racing here! */ ret = smsc75xx_write_reg(dev, RFE_CTL, pdata->rfe_ctl); if (ret < 0) { netdev_warn(dev->net, "Error writing RFE_CTL\n"); return ret; } return 0; } static int smsc75xx_wait_ready(struct usbnet *dev, int in_pm) { int timeout = 0; do { u32 buf; int ret; ret = __smsc75xx_read_reg(dev, PMT_CTL, &buf, in_pm); if (ret < 0) { netdev_warn(dev->net, "Failed to read PMT_CTL: %d\n", ret); return ret; } if (buf & PMT_CTL_DEV_RDY) return 0; msleep(10); timeout++; } while (timeout < 100); netdev_warn(dev->net, "timeout waiting for device ready\n"); return -EIO; } static int smsc75xx_phy_gig_workaround(struct usbnet *dev) { struct mii_if_info *mii = &dev->mii; int ret = 0, timeout = 0; u32 buf, link_up = 0; /* Set the phy in Gig loopback */ smsc75xx_mdio_write(dev->net, mii->phy_id, MII_BMCR, 0x4040); /* Wait for the link up */ do { link_up = smsc75xx_link_ok_nopm(dev); usleep_range(10000, 20000); timeout++; } while ((!link_up) && (timeout < 1000)); if (timeout >= 1000) { netdev_warn(dev->net, "Timeout waiting for PHY link up\n"); return -EIO; } /* phy reset */ ret = smsc75xx_read_reg(dev, PMT_CTL, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read PMT_CTL: %d\n", ret); return ret; } buf |= PMT_CTL_PHY_RST; ret = smsc75xx_write_reg(dev, PMT_CTL, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write PMT_CTL: %d\n", ret); return ret; } timeout = 0; do { usleep_range(10000, 20000); ret = smsc75xx_read_reg(dev, PMT_CTL, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read PMT_CTL: %d\n", ret); return ret; } timeout++; } while ((buf & PMT_CTL_PHY_RST) && (timeout < 100)); if (timeout >= 100) { netdev_warn(dev->net, "timeout waiting for PHY Reset\n"); return -EIO; } return 0; } static int smsc75xx_reset(struct usbnet *dev) { struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); u32 buf; int ret = 0, timeout; netif_dbg(dev, ifup, dev->net, "entering smsc75xx_reset\n"); ret = smsc75xx_wait_ready(dev, 0); if (ret < 0) { netdev_warn(dev->net, "device not ready in smsc75xx_reset\n"); return ret; } ret = smsc75xx_read_reg(dev, HW_CFG, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read HW_CFG: %d\n", ret); return ret; } buf |= HW_CFG_LRST; ret = smsc75xx_write_reg(dev, HW_CFG, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write HW_CFG: %d\n", ret); return ret; } timeout = 0; do { msleep(10); ret = smsc75xx_read_reg(dev, HW_CFG, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read HW_CFG: %d\n", ret); return ret; } timeout++; } while ((buf & HW_CFG_LRST) && (timeout < 100)); if (timeout >= 100) { netdev_warn(dev->net, "timeout on completion of Lite Reset\n"); return -EIO; } netif_dbg(dev, ifup, dev->net, "Lite reset complete, resetting PHY\n"); ret = smsc75xx_read_reg(dev, PMT_CTL, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read PMT_CTL: %d\n", ret); return ret; } buf |= PMT_CTL_PHY_RST; ret = smsc75xx_write_reg(dev, PMT_CTL, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write PMT_CTL: %d\n", ret); return ret; } timeout = 0; do { msleep(10); ret = smsc75xx_read_reg(dev, PMT_CTL, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read PMT_CTL: %d\n", ret); return ret; } timeout++; } while ((buf & PMT_CTL_PHY_RST) && (timeout < 100)); if (timeout >= 100) { netdev_warn(dev->net, "timeout waiting for PHY Reset\n"); return -EIO; } netif_dbg(dev, ifup, dev->net, "PHY reset complete\n"); ret = smsc75xx_set_mac_address(dev); if (ret < 0) { netdev_warn(dev->net, "Failed to set mac address\n"); return ret; } netif_dbg(dev, ifup, dev->net, "MAC Address: %pM\n", dev->net->dev_addr); ret = smsc75xx_read_reg(dev, HW_CFG, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read HW_CFG: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "Read Value from HW_CFG : 0x%08x\n", buf); buf |= HW_CFG_BIR; ret = smsc75xx_write_reg(dev, HW_CFG, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write HW_CFG: %d\n", ret); return ret; } ret = smsc75xx_read_reg(dev, HW_CFG, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read HW_CFG: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "Read Value from HW_CFG after writing HW_CFG_BIR: 0x%08x\n", buf); if (!turbo_mode) { buf = 0; dev->rx_urb_size = MAX_SINGLE_PACKET_SIZE; } else if (dev->udev->speed == USB_SPEED_HIGH) { buf = DEFAULT_HS_BURST_CAP_SIZE / HS_USB_PKT_SIZE; dev->rx_urb_size = DEFAULT_HS_BURST_CAP_SIZE; } else { buf = DEFAULT_FS_BURST_CAP_SIZE / FS_USB_PKT_SIZE; dev->rx_urb_size = DEFAULT_FS_BURST_CAP_SIZE; } netif_dbg(dev, ifup, dev->net, "rx_urb_size=%ld\n", (ulong)dev->rx_urb_size); ret = smsc75xx_write_reg(dev, BURST_CAP, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write BURST_CAP: %d\n", ret); return ret; } ret = smsc75xx_read_reg(dev, BURST_CAP, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read BURST_CAP: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "Read Value from BURST_CAP after writing: 0x%08x\n", buf); ret = smsc75xx_write_reg(dev, BULK_IN_DLY, DEFAULT_BULK_IN_DELAY); if (ret < 0) { netdev_warn(dev->net, "Failed to write BULK_IN_DLY: %d\n", ret); return ret; } ret = smsc75xx_read_reg(dev, BULK_IN_DLY, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read BULK_IN_DLY: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "Read Value from BULK_IN_DLY after writing: 0x%08x\n", buf); if (turbo_mode) { ret = smsc75xx_read_reg(dev, HW_CFG, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read HW_CFG: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "HW_CFG: 0x%08x\n", buf); buf |= (HW_CFG_MEF | HW_CFG_BCE); ret = smsc75xx_write_reg(dev, HW_CFG, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write HW_CFG: %d\n", ret); return ret; } ret = smsc75xx_read_reg(dev, HW_CFG, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read HW_CFG: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "HW_CFG: 0x%08x\n", buf); } /* set FIFO sizes */ buf = (MAX_RX_FIFO_SIZE - 512) / 512; ret = smsc75xx_write_reg(dev, FCT_RX_FIFO_END, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write FCT_RX_FIFO_END: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "FCT_RX_FIFO_END set to 0x%08x\n", buf); buf = (MAX_TX_FIFO_SIZE - 512) / 512; ret = smsc75xx_write_reg(dev, FCT_TX_FIFO_END, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write FCT_TX_FIFO_END: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "FCT_TX_FIFO_END set to 0x%08x\n", buf); ret = smsc75xx_write_reg(dev, INT_STS, INT_STS_CLEAR_ALL); if (ret < 0) { netdev_warn(dev->net, "Failed to write INT_STS: %d\n", ret); return ret; } ret = smsc75xx_read_reg(dev, ID_REV, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read ID_REV: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "ID_REV = 0x%08x\n", buf); ret = smsc75xx_read_reg(dev, E2P_CMD, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read E2P_CMD: %d\n", ret); return ret; } /* only set default GPIO/LED settings if no EEPROM is detected */ if (!(buf & E2P_CMD_LOADED)) { ret = smsc75xx_read_reg(dev, LED_GPIO_CFG, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read LED_GPIO_CFG: %d\n", ret); return ret; } buf &= ~(LED_GPIO_CFG_LED2_FUN_SEL | LED_GPIO_CFG_LED10_FUN_SEL); buf |= LED_GPIO_CFG_LEDGPIO_EN | LED_GPIO_CFG_LED2_FUN_SEL; ret = smsc75xx_write_reg(dev, LED_GPIO_CFG, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write LED_GPIO_CFG: %d\n", ret); return ret; } } ret = smsc75xx_write_reg(dev, FLOW, 0); if (ret < 0) { netdev_warn(dev->net, "Failed to write FLOW: %d\n", ret); return ret; } ret = smsc75xx_write_reg(dev, FCT_FLOW, 0); if (ret < 0) { netdev_warn(dev->net, "Failed to write FCT_FLOW: %d\n", ret); return ret; } /* Don't need rfe_ctl_lock during initialisation */ ret = smsc75xx_read_reg(dev, RFE_CTL, &pdata->rfe_ctl); if (ret < 0) { netdev_warn(dev->net, "Failed to read RFE_CTL: %d\n", ret); return ret; } pdata->rfe_ctl |= RFE_CTL_AB | RFE_CTL_DPF; ret = smsc75xx_write_reg(dev, RFE_CTL, pdata->rfe_ctl); if (ret < 0) { netdev_warn(dev->net, "Failed to write RFE_CTL: %d\n", ret); return ret; } ret = smsc75xx_read_reg(dev, RFE_CTL, &pdata->rfe_ctl); if (ret < 0) { netdev_warn(dev->net, "Failed to read RFE_CTL: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "RFE_CTL set to 0x%08x\n", pdata->rfe_ctl); /* Enable or disable checksum offload engines */ smsc75xx_set_features(dev->net, dev->net->features); smsc75xx_set_multicast(dev->net); ret = smsc75xx_phy_initialize(dev); if (ret < 0) { netdev_warn(dev->net, "Failed to initialize PHY: %d\n", ret); return ret; } ret = smsc75xx_read_reg(dev, INT_EP_CTL, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read INT_EP_CTL: %d\n", ret); return ret; } /* enable PHY interrupts */ buf |= INT_ENP_PHY_INT; ret = smsc75xx_write_reg(dev, INT_EP_CTL, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write INT_EP_CTL: %d\n", ret); return ret; } /* allow mac to detect speed and duplex from phy */ ret = smsc75xx_read_reg(dev, MAC_CR, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read MAC_CR: %d\n", ret); return ret; } buf |= (MAC_CR_ADD | MAC_CR_ASD); ret = smsc75xx_write_reg(dev, MAC_CR, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write MAC_CR: %d\n", ret); return ret; } ret = smsc75xx_read_reg(dev, MAC_TX, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read MAC_TX: %d\n", ret); return ret; } buf |= MAC_TX_TXEN; ret = smsc75xx_write_reg(dev, MAC_TX, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write MAC_TX: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "MAC_TX set to 0x%08x\n", buf); ret = smsc75xx_read_reg(dev, FCT_TX_CTL, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read FCT_TX_CTL: %d\n", ret); return ret; } buf |= FCT_TX_CTL_EN; ret = smsc75xx_write_reg(dev, FCT_TX_CTL, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write FCT_TX_CTL: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "FCT_TX_CTL set to 0x%08x\n", buf); ret = smsc75xx_set_rx_max_frame_length(dev, dev->net->mtu + ETH_HLEN); if (ret < 0) { netdev_warn(dev->net, "Failed to set max rx frame length\n"); return ret; } ret = smsc75xx_read_reg(dev, MAC_RX, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read MAC_RX: %d\n", ret); return ret; } buf |= MAC_RX_RXEN; ret = smsc75xx_write_reg(dev, MAC_RX, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write MAC_RX: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "MAC_RX set to 0x%08x\n", buf); ret = smsc75xx_read_reg(dev, FCT_RX_CTL, &buf); if (ret < 0) { netdev_warn(dev->net, "Failed to read FCT_RX_CTL: %d\n", ret); return ret; } buf |= FCT_RX_CTL_EN; ret = smsc75xx_write_reg(dev, FCT_RX_CTL, buf); if (ret < 0) { netdev_warn(dev->net, "Failed to write FCT_RX_CTL: %d\n", ret); return ret; } netif_dbg(dev, ifup, dev->net, "FCT_RX_CTL set to 0x%08x\n", buf); netif_dbg(dev, ifup, dev->net, "smsc75xx_reset, return 0\n"); return 0; } static const struct net_device_ops smsc75xx_netdev_ops = { .ndo_open = usbnet_open, .ndo_stop = usbnet_stop, .ndo_start_xmit = usbnet_start_xmit, .ndo_tx_timeout = usbnet_tx_timeout, .ndo_get_stats64 = dev_get_tstats64, .ndo_change_mtu = smsc75xx_change_mtu, .ndo_set_mac_address = eth_mac_addr, .ndo_validate_addr = eth_validate_addr, .ndo_eth_ioctl = smsc75xx_ioctl, .ndo_set_rx_mode = smsc75xx_set_multicast, .ndo_set_features = smsc75xx_set_features, }; static int smsc75xx_bind(struct usbnet *dev, struct usb_interface *intf) { struct smsc75xx_priv *pdata = NULL; int ret; printk(KERN_INFO SMSC_CHIPNAME " v" SMSC_DRIVER_VERSION "\n"); ret = usbnet_get_endpoints(dev, intf); if (ret < 0) { netdev_warn(dev->net, "usbnet_get_endpoints failed: %d\n", ret); return ret; } dev->data[0] = (unsigned long)kzalloc(sizeof(struct smsc75xx_priv), GFP_KERNEL); pdata = (struct smsc75xx_priv *)(dev->data[0]); if (!pdata) return -ENOMEM; pdata->dev = dev; spin_lock_init(&pdata->rfe_ctl_lock); mutex_init(&pdata->dataport_mutex); INIT_WORK(&pdata->set_multicast, smsc75xx_deferred_multicast_write); if (DEFAULT_TX_CSUM_ENABLE) dev->net->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; if (DEFAULT_RX_CSUM_ENABLE) dev->net->features |= NETIF_F_RXCSUM; dev->net->hw_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM; ret = smsc75xx_wait_ready(dev, 0); if (ret < 0) { netdev_warn(dev->net, "device not ready in smsc75xx_bind\n"); goto free_pdata; } smsc75xx_init_mac_address(dev); /* Init all registers */ ret = smsc75xx_reset(dev); if (ret < 0) { netdev_warn(dev->net, "smsc75xx_reset error %d\n", ret); goto cancel_work; } dev->net->netdev_ops = &smsc75xx_netdev_ops; dev->net->ethtool_ops = &smsc75xx_ethtool_ops; dev->net->flags |= IFF_MULTICAST; dev->net->hard_header_len += SMSC75XX_TX_OVERHEAD; dev->hard_mtu = dev->net->mtu + dev->net->hard_header_len; dev->net->max_mtu = MAX_SINGLE_PACKET_SIZE; return 0; cancel_work: cancel_work_sync(&pdata->set_multicast); free_pdata: kfree(pdata); dev->data[0] = 0; return ret; } static void smsc75xx_unbind(struct usbnet *dev, struct usb_interface *intf) { struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); if (pdata) { cancel_work_sync(&pdata->set_multicast); netif_dbg(dev, ifdown, dev->net, "free pdata\n"); kfree(pdata); dev->data[0] = 0; } } static u16 smsc_crc(const u8 *buffer, size_t len) { return bitrev16(crc16(0xFFFF, buffer, len)); } static int smsc75xx_write_wuff(struct usbnet *dev, int filter, u32 wuf_cfg, u32 wuf_mask1) { int cfg_base = WUF_CFGX + filter * 4; int mask_base = WUF_MASKX + filter * 16; int ret; ret = smsc75xx_write_reg(dev, cfg_base, wuf_cfg); if (ret < 0) { netdev_warn(dev->net, "Error writing WUF_CFGX\n"); return ret; } ret = smsc75xx_write_reg(dev, mask_base, wuf_mask1); if (ret < 0) { netdev_warn(dev->net, "Error writing WUF_MASKX\n"); return ret; } ret = smsc75xx_write_reg(dev, mask_base + 4, 0); if (ret < 0) { netdev_warn(dev->net, "Error writing WUF_MASKX\n"); return ret; } ret = smsc75xx_write_reg(dev, mask_base + 8, 0); if (ret < 0) { netdev_warn(dev->net, "Error writing WUF_MASKX\n"); return ret; } ret = smsc75xx_write_reg(dev, mask_base + 12, 0); if (ret < 0) { netdev_warn(dev->net, "Error writing WUF_MASKX\n"); return ret; } return 0; } static int smsc75xx_enter_suspend0(struct usbnet *dev) { struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); u32 val; int ret; ret = smsc75xx_read_reg_nopm(dev, PMT_CTL, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading PMT_CTL\n"); return ret; } val &= (~(PMT_CTL_SUS_MODE | PMT_CTL_PHY_RST)); val |= PMT_CTL_SUS_MODE_0 | PMT_CTL_WOL_EN | PMT_CTL_WUPS; ret = smsc75xx_write_reg_nopm(dev, PMT_CTL, val); if (ret < 0) { netdev_warn(dev->net, "Error writing PMT_CTL\n"); return ret; } pdata->suspend_flags |= SUSPEND_SUSPEND0; return 0; } static int smsc75xx_enter_suspend1(struct usbnet *dev) { struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); u32 val; int ret; ret = smsc75xx_read_reg_nopm(dev, PMT_CTL, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading PMT_CTL\n"); return ret; } val &= ~(PMT_CTL_SUS_MODE | PMT_CTL_WUPS | PMT_CTL_PHY_RST); val |= PMT_CTL_SUS_MODE_1; ret = smsc75xx_write_reg_nopm(dev, PMT_CTL, val); if (ret < 0) { netdev_warn(dev->net, "Error writing PMT_CTL\n"); return ret; } /* clear wol status, enable energy detection */ val &= ~PMT_CTL_WUPS; val |= (PMT_CTL_WUPS_ED | PMT_CTL_ED_EN); ret = smsc75xx_write_reg_nopm(dev, PMT_CTL, val); if (ret < 0) { netdev_warn(dev->net, "Error writing PMT_CTL\n"); return ret; } pdata->suspend_flags |= SUSPEND_SUSPEND1; return 0; } static int smsc75xx_enter_suspend2(struct usbnet *dev) { struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); u32 val; int ret; ret = smsc75xx_read_reg_nopm(dev, PMT_CTL, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading PMT_CTL\n"); return ret; } val &= ~(PMT_CTL_SUS_MODE | PMT_CTL_WUPS | PMT_CTL_PHY_RST); val |= PMT_CTL_SUS_MODE_2; ret = smsc75xx_write_reg_nopm(dev, PMT_CTL, val); if (ret < 0) { netdev_warn(dev->net, "Error writing PMT_CTL\n"); return ret; } pdata->suspend_flags |= SUSPEND_SUSPEND2; return 0; } static int smsc75xx_enter_suspend3(struct usbnet *dev) { struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); u32 val; int ret; ret = smsc75xx_read_reg_nopm(dev, FCT_RX_CTL, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading FCT_RX_CTL\n"); return ret; } if (val & FCT_RX_CTL_RXUSED) { netdev_dbg(dev->net, "rx fifo not empty in autosuspend\n"); return -EBUSY; } ret = smsc75xx_read_reg_nopm(dev, PMT_CTL, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading PMT_CTL\n"); return ret; } val &= ~(PMT_CTL_SUS_MODE | PMT_CTL_WUPS | PMT_CTL_PHY_RST); val |= PMT_CTL_SUS_MODE_3 | PMT_CTL_RES_CLR_WKP_EN; ret = smsc75xx_write_reg_nopm(dev, PMT_CTL, val); if (ret < 0) { netdev_warn(dev->net, "Error writing PMT_CTL\n"); return ret; } /* clear wol status */ val &= ~PMT_CTL_WUPS; val |= PMT_CTL_WUPS_WOL; ret = smsc75xx_write_reg_nopm(dev, PMT_CTL, val); if (ret < 0) { netdev_warn(dev->net, "Error writing PMT_CTL\n"); return ret; } pdata->suspend_flags |= SUSPEND_SUSPEND3; return 0; } static int smsc75xx_enable_phy_wakeup_interrupts(struct usbnet *dev, u16 mask) { struct mii_if_info *mii = &dev->mii; int ret; netdev_dbg(dev->net, "enabling PHY wakeup interrupts\n"); /* read to clear */ ret = smsc75xx_mdio_read_nopm(dev->net, mii->phy_id, PHY_INT_SRC); if (ret < 0) { netdev_warn(dev->net, "Error reading PHY_INT_SRC\n"); return ret; } /* enable interrupt source */ ret = smsc75xx_mdio_read_nopm(dev->net, mii->phy_id, PHY_INT_MASK); if (ret < 0) { netdev_warn(dev->net, "Error reading PHY_INT_MASK\n"); return ret; } ret |= mask; smsc75xx_mdio_write_nopm(dev->net, mii->phy_id, PHY_INT_MASK, ret); return 0; } static int smsc75xx_link_ok_nopm(struct usbnet *dev) { struct mii_if_info *mii = &dev->mii; int ret; /* first, a dummy read, needed to latch some MII phys */ ret = smsc75xx_mdio_read_nopm(dev->net, mii->phy_id, MII_BMSR); if (ret < 0) { netdev_warn(dev->net, "Error reading MII_BMSR\n"); return ret; } ret = smsc75xx_mdio_read_nopm(dev->net, mii->phy_id, MII_BMSR); if (ret < 0) { netdev_warn(dev->net, "Error reading MII_BMSR\n"); return ret; } return !!(ret & BMSR_LSTATUS); } static int smsc75xx_autosuspend(struct usbnet *dev, u32 link_up) { int ret; if (!netif_running(dev->net)) { /* interface is ifconfig down so fully power down hw */ netdev_dbg(dev->net, "autosuspend entering SUSPEND2\n"); return smsc75xx_enter_suspend2(dev); } if (!link_up) { /* link is down so enter EDPD mode */ netdev_dbg(dev->net, "autosuspend entering SUSPEND1\n"); /* enable PHY wakeup events for if cable is attached */ ret = smsc75xx_enable_phy_wakeup_interrupts(dev, PHY_INT_MASK_ANEG_COMP); if (ret < 0) { netdev_warn(dev->net, "error enabling PHY wakeup ints\n"); return ret; } netdev_info(dev->net, "entering SUSPEND1 mode\n"); return smsc75xx_enter_suspend1(dev); } /* enable PHY wakeup events so we remote wakeup if cable is pulled */ ret = smsc75xx_enable_phy_wakeup_interrupts(dev, PHY_INT_MASK_LINK_DOWN); if (ret < 0) { netdev_warn(dev->net, "error enabling PHY wakeup ints\n"); return ret; } netdev_dbg(dev->net, "autosuspend entering SUSPEND3\n"); return smsc75xx_enter_suspend3(dev); } static int smsc75xx_suspend(struct usb_interface *intf, pm_message_t message) { struct usbnet *dev = usb_get_intfdata(intf); struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); u32 val, link_up; int ret; ret = usbnet_suspend(intf, message); if (ret < 0) { netdev_warn(dev->net, "usbnet_suspend error\n"); return ret; } if (pdata->suspend_flags) { netdev_warn(dev->net, "error during last resume\n"); pdata->suspend_flags = 0; } /* determine if link is up using only _nopm functions */ link_up = smsc75xx_link_ok_nopm(dev); if (message.event == PM_EVENT_AUTO_SUSPEND) { ret = smsc75xx_autosuspend(dev, link_up); goto done; } /* if we get this far we're not autosuspending */ /* if no wol options set, or if link is down and we're not waking on * PHY activity, enter lowest power SUSPEND2 mode */ if (!(pdata->wolopts & SUPPORTED_WAKE) || !(link_up || (pdata->wolopts & WAKE_PHY))) { netdev_info(dev->net, "entering SUSPEND2 mode\n"); /* disable energy detect (link up) & wake up events */ ret = smsc75xx_read_reg_nopm(dev, WUCSR, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading WUCSR\n"); goto done; } val &= ~(WUCSR_MPEN | WUCSR_WUEN); ret = smsc75xx_write_reg_nopm(dev, WUCSR, val); if (ret < 0) { netdev_warn(dev->net, "Error writing WUCSR\n"); goto done; } ret = smsc75xx_read_reg_nopm(dev, PMT_CTL, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading PMT_CTL\n"); goto done; } val &= ~(PMT_CTL_ED_EN | PMT_CTL_WOL_EN); ret = smsc75xx_write_reg_nopm(dev, PMT_CTL, val); if (ret < 0) { netdev_warn(dev->net, "Error writing PMT_CTL\n"); goto done; } ret = smsc75xx_enter_suspend2(dev); goto done; } if (pdata->wolopts & WAKE_PHY) { ret = smsc75xx_enable_phy_wakeup_interrupts(dev, (PHY_INT_MASK_ANEG_COMP | PHY_INT_MASK_LINK_DOWN)); if (ret < 0) { netdev_warn(dev->net, "error enabling PHY wakeup ints\n"); goto done; } /* if link is down then configure EDPD and enter SUSPEND1, * otherwise enter SUSPEND0 below */ if (!link_up) { struct mii_if_info *mii = &dev->mii; netdev_info(dev->net, "entering SUSPEND1 mode\n"); /* enable energy detect power-down mode */ ret = smsc75xx_mdio_read_nopm(dev->net, mii->phy_id, PHY_MODE_CTRL_STS); if (ret < 0) { netdev_warn(dev->net, "Error reading PHY_MODE_CTRL_STS\n"); goto done; } ret |= MODE_CTRL_STS_EDPWRDOWN; smsc75xx_mdio_write_nopm(dev->net, mii->phy_id, PHY_MODE_CTRL_STS, ret); /* enter SUSPEND1 mode */ ret = smsc75xx_enter_suspend1(dev); goto done; } } if (pdata->wolopts & (WAKE_MCAST | WAKE_ARP)) { int i, filter = 0; /* disable all filters */ for (i = 0; i < WUF_NUM; i++) { ret = smsc75xx_write_reg_nopm(dev, WUF_CFGX + i * 4, 0); if (ret < 0) { netdev_warn(dev->net, "Error writing WUF_CFGX\n"); goto done; } } if (pdata->wolopts & WAKE_MCAST) { const u8 mcast[] = {0x01, 0x00, 0x5E}; netdev_info(dev->net, "enabling multicast detection\n"); val = WUF_CFGX_EN | WUF_CFGX_ATYPE_MULTICAST | smsc_crc(mcast, 3); ret = smsc75xx_write_wuff(dev, filter++, val, 0x0007); if (ret < 0) { netdev_warn(dev->net, "Error writing wakeup filter\n"); goto done; } } if (pdata->wolopts & WAKE_ARP) { const u8 arp[] = {0x08, 0x06}; netdev_info(dev->net, "enabling ARP detection\n"); val = WUF_CFGX_EN | WUF_CFGX_ATYPE_ALL | (0x0C << 16) | smsc_crc(arp, 2); ret = smsc75xx_write_wuff(dev, filter++, val, 0x0003); if (ret < 0) { netdev_warn(dev->net, "Error writing wakeup filter\n"); goto done; } } /* clear any pending pattern match packet status */ ret = smsc75xx_read_reg_nopm(dev, WUCSR, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading WUCSR\n"); goto done; } val |= WUCSR_WUFR; ret = smsc75xx_write_reg_nopm(dev, WUCSR, val); if (ret < 0) { netdev_warn(dev->net, "Error writing WUCSR\n"); goto done; } netdev_info(dev->net, "enabling packet match detection\n"); ret = smsc75xx_read_reg_nopm(dev, WUCSR, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading WUCSR\n"); goto done; } val |= WUCSR_WUEN; ret = smsc75xx_write_reg_nopm(dev, WUCSR, val); if (ret < 0) { netdev_warn(dev->net, "Error writing WUCSR\n"); goto done; } } else { netdev_info(dev->net, "disabling packet match detection\n"); ret = smsc75xx_read_reg_nopm(dev, WUCSR, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading WUCSR\n"); goto done; } val &= ~WUCSR_WUEN; ret = smsc75xx_write_reg_nopm(dev, WUCSR, val); if (ret < 0) { netdev_warn(dev->net, "Error writing WUCSR\n"); goto done; } } /* disable magic, bcast & unicast wakeup sources */ ret = smsc75xx_read_reg_nopm(dev, WUCSR, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading WUCSR\n"); goto done; } val &= ~(WUCSR_MPEN | WUCSR_BCST_EN | WUCSR_PFDA_EN); ret = smsc75xx_write_reg_nopm(dev, WUCSR, val); if (ret < 0) { netdev_warn(dev->net, "Error writing WUCSR\n"); goto done; } if (pdata->wolopts & WAKE_PHY) { netdev_info(dev->net, "enabling PHY wakeup\n"); ret = smsc75xx_read_reg_nopm(dev, PMT_CTL, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading PMT_CTL\n"); goto done; } /* clear wol status, enable energy detection */ val &= ~PMT_CTL_WUPS; val |= (PMT_CTL_WUPS_ED | PMT_CTL_ED_EN); ret = smsc75xx_write_reg_nopm(dev, PMT_CTL, val); if (ret < 0) { netdev_warn(dev->net, "Error writing PMT_CTL\n"); goto done; } } if (pdata->wolopts & WAKE_MAGIC) { netdev_info(dev->net, "enabling magic packet wakeup\n"); ret = smsc75xx_read_reg_nopm(dev, WUCSR, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading WUCSR\n"); goto done; } /* clear any pending magic packet status */ val |= WUCSR_MPR | WUCSR_MPEN; ret = smsc75xx_write_reg_nopm(dev, WUCSR, val); if (ret < 0) { netdev_warn(dev->net, "Error writing WUCSR\n"); goto done; } } if (pdata->wolopts & WAKE_BCAST) { netdev_info(dev->net, "enabling broadcast detection\n"); ret = smsc75xx_read_reg_nopm(dev, WUCSR, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading WUCSR\n"); goto done; } val |= WUCSR_BCAST_FR | WUCSR_BCST_EN; ret = smsc75xx_write_reg_nopm(dev, WUCSR, val); if (ret < 0) { netdev_warn(dev->net, "Error writing WUCSR\n"); goto done; } } if (pdata->wolopts & WAKE_UCAST) { netdev_info(dev->net, "enabling unicast detection\n"); ret = smsc75xx_read_reg_nopm(dev, WUCSR, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading WUCSR\n"); goto done; } val |= WUCSR_WUFR | WUCSR_PFDA_EN; ret = smsc75xx_write_reg_nopm(dev, WUCSR, val); if (ret < 0) { netdev_warn(dev->net, "Error writing WUCSR\n"); goto done; } } /* enable receiver to enable frame reception */ ret = smsc75xx_read_reg_nopm(dev, MAC_RX, &val); if (ret < 0) { netdev_warn(dev->net, "Failed to read MAC_RX: %d\n", ret); goto done; } val |= MAC_RX_RXEN; ret = smsc75xx_write_reg_nopm(dev, MAC_RX, val); if (ret < 0) { netdev_warn(dev->net, "Failed to write MAC_RX: %d\n", ret); goto done; } /* some wol options are enabled, so enter SUSPEND0 */ netdev_info(dev->net, "entering SUSPEND0 mode\n"); ret = smsc75xx_enter_suspend0(dev); done: /* * TODO: resume() might need to handle the suspend failure * in system sleep */ if (ret && PMSG_IS_AUTO(message)) usbnet_resume(intf); return ret; } static int smsc75xx_resume(struct usb_interface *intf) { struct usbnet *dev = usb_get_intfdata(intf); struct smsc75xx_priv *pdata = (struct smsc75xx_priv *)(dev->data[0]); u8 suspend_flags = pdata->suspend_flags; int ret; u32 val; netdev_dbg(dev->net, "resume suspend_flags=0x%02x\n", suspend_flags); /* do this first to ensure it's cleared even in error case */ pdata->suspend_flags = 0; if (suspend_flags & SUSPEND_ALLMODES) { /* Disable wakeup sources */ ret = smsc75xx_read_reg_nopm(dev, WUCSR, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading WUCSR\n"); return ret; } val &= ~(WUCSR_WUEN | WUCSR_MPEN | WUCSR_PFDA_EN | WUCSR_BCST_EN); ret = smsc75xx_write_reg_nopm(dev, WUCSR, val); if (ret < 0) { netdev_warn(dev->net, "Error writing WUCSR\n"); return ret; } /* clear wake-up status */ ret = smsc75xx_read_reg_nopm(dev, PMT_CTL, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading PMT_CTL\n"); return ret; } val &= ~PMT_CTL_WOL_EN; val |= PMT_CTL_WUPS; ret = smsc75xx_write_reg_nopm(dev, PMT_CTL, val); if (ret < 0) { netdev_warn(dev->net, "Error writing PMT_CTL\n"); return ret; } } if (suspend_flags & SUSPEND_SUSPEND2) { netdev_info(dev->net, "resuming from SUSPEND2\n"); ret = smsc75xx_read_reg_nopm(dev, PMT_CTL, &val); if (ret < 0) { netdev_warn(dev->net, "Error reading PMT_CTL\n"); return ret; } val |= PMT_CTL_PHY_PWRUP; ret = smsc75xx_write_reg_nopm(dev, PMT_CTL, val); if (ret < 0) { netdev_warn(dev->net, "Error writing PMT_CTL\n"); return ret; } } ret = smsc75xx_wait_ready(dev, 1); if (ret < 0) { netdev_warn(dev->net, "device not ready in smsc75xx_resume\n"); return ret; } return usbnet_resume(intf); } static void smsc75xx_rx_csum_offload(struct usbnet *dev, struct sk_buff *skb, u32 rx_cmd_a, u32 rx_cmd_b) { if (!(dev->net->features & NETIF_F_RXCSUM) || unlikely(rx_cmd_a & RX_CMD_A_LCSM)) { skb->ip_summed = CHECKSUM_NONE; } else { skb->csum = ntohs((u16)(rx_cmd_b >> RX_CMD_B_CSUM_SHIFT)); skb->ip_summed = CHECKSUM_COMPLETE; } } static int smsc75xx_rx_fixup(struct usbnet *dev, struct sk_buff *skb) { /* This check is no longer done by usbnet */ if (skb->len < dev->net->hard_header_len) return 0; while (skb->len > 0) { u32 rx_cmd_a, rx_cmd_b, align_count, size; struct sk_buff *ax_skb; unsigned char *packet; rx_cmd_a = get_unaligned_le32(skb->data); skb_pull(skb, 4); rx_cmd_b = get_unaligned_le32(skb->data); skb_pull(skb, 4 + RXW_PADDING); packet = skb->data; /* get the packet length */ size = (rx_cmd_a & RX_CMD_A_LEN) - RXW_PADDING; align_count = (4 - ((size + RXW_PADDING) % 4)) % 4; if (unlikely(size > skb->len)) { netif_dbg(dev, rx_err, dev->net, "size err rx_cmd_a=0x%08x\n", rx_cmd_a); return 0; } if (unlikely(rx_cmd_a & RX_CMD_A_RED)) { netif_dbg(dev, rx_err, dev->net, "Error rx_cmd_a=0x%08x\n", rx_cmd_a); dev->net->stats.rx_errors++; dev->net->stats.rx_dropped++; if (rx_cmd_a & RX_CMD_A_FCS) dev->net->stats.rx_crc_errors++; else if (rx_cmd_a & (RX_CMD_A_LONG | RX_CMD_A_RUNT)) dev->net->stats.rx_frame_errors++; } else { /* MAX_SINGLE_PACKET_SIZE + 4(CRC) + 2(COE) + 4(Vlan) */ if (unlikely(size > (MAX_SINGLE_PACKET_SIZE + ETH_HLEN + 12))) { netif_dbg(dev, rx_err, dev->net, "size err rx_cmd_a=0x%08x\n", rx_cmd_a); return 0; } /* last frame in this batch */ if (skb->len == size) { smsc75xx_rx_csum_offload(dev, skb, rx_cmd_a, rx_cmd_b); skb_trim(skb, skb->len - 4); /* remove fcs */ return 1; } /* Use "size - 4" to remove fcs */ ax_skb = netdev_alloc_skb_ip_align(dev->net, size - 4); if (unlikely(!ax_skb)) { netdev_warn(dev->net, "Error allocating skb\n"); return 0; } skb_put(ax_skb, size - 4); memcpy(ax_skb->data, packet, size - 4); smsc75xx_rx_csum_offload(dev, ax_skb, rx_cmd_a, rx_cmd_b); usbnet_skb_return(dev, ax_skb); } skb_pull(skb, size); /* padding bytes before the next frame starts */ if (skb->len) skb_pull(skb, align_count); } return 1; } static struct sk_buff *smsc75xx_tx_fixup(struct usbnet *dev, struct sk_buff *skb, gfp_t flags) { u32 tx_cmd_a, tx_cmd_b; void *ptr; if (skb_cow_head(skb, SMSC75XX_TX_OVERHEAD)) { dev_kfree_skb_any(skb); return NULL; } tx_cmd_a = (u32)(skb->len & TX_CMD_A_LEN) | TX_CMD_A_FCS; if (skb->ip_summed == CHECKSUM_PARTIAL) tx_cmd_a |= TX_CMD_A_IPE | TX_CMD_A_TPE; if (skb_is_gso(skb)) { u16 mss = max(skb_shinfo(skb)->gso_size, TX_MSS_MIN); tx_cmd_b = (mss << TX_CMD_B_MSS_SHIFT) & TX_CMD_B_MSS; tx_cmd_a |= TX_CMD_A_LSO; } else { tx_cmd_b = 0; } ptr = skb_push(skb, 8); put_unaligned_le32(tx_cmd_a, ptr); put_unaligned_le32(tx_cmd_b, ptr + 4); return skb; } static int smsc75xx_manage_power(struct usbnet *dev, int on) { dev->intf->needs_remote_wakeup = on; return 0; } static const struct driver_info smsc75xx_info = { .description = "smsc75xx USB 2.0 Gigabit Ethernet", .bind = smsc75xx_bind, .unbind = smsc75xx_unbind, .link_reset = smsc75xx_link_reset, .reset = smsc75xx_reset, .rx_fixup = smsc75xx_rx_fixup, .tx_fixup = smsc75xx_tx_fixup, .status = smsc75xx_status, .manage_power = smsc75xx_manage_power, .flags = FLAG_ETHER | FLAG_SEND_ZLP | FLAG_LINK_INTR, }; static const struct usb_device_id products[] = { { /* SMSC7500 USB Gigabit Ethernet Device */ USB_DEVICE(USB_VENDOR_ID_SMSC, USB_PRODUCT_ID_LAN7500), .driver_info = (unsigned long) &smsc75xx_info, }, { /* SMSC7500 USB Gigabit Ethernet Device */ USB_DEVICE(USB_VENDOR_ID_SMSC, USB_PRODUCT_ID_LAN7505), .driver_info = (unsigned long) &smsc75xx_info, }, { }, /* END */ }; MODULE_DEVICE_TABLE(usb, products); static struct usb_driver smsc75xx_driver = { .name = SMSC_CHIPNAME, .id_table = products, .probe = usbnet_probe, .suspend = smsc75xx_suspend, .resume = smsc75xx_resume, .reset_resume = smsc75xx_resume, .disconnect = usbnet_disconnect, .disable_hub_initiated_lpm = 1, .supports_autosuspend = 1, }; module_usb_driver(smsc75xx_driver); MODULE_AUTHOR("Nancy Lin"); MODULE_AUTHOR("Steve Glendinning <steve.glendinning@shawell.net>"); MODULE_DESCRIPTION("SMSC75XX USB 2.0 Gigabit Ethernet Devices"); MODULE_LICENSE("GPL"); |
| 27 27 8 21 21 21 21 27 27 27 39 39 21 6 39 39 10 10 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 | // SPDX-License-Identifier: GPL-2.0-only /* Copyright (c) 2016 Facebook */ #include "percpu_freelist.h" int pcpu_freelist_init(struct pcpu_freelist *s) { int cpu; s->freelist = alloc_percpu(struct pcpu_freelist_head); if (!s->freelist) return -ENOMEM; for_each_possible_cpu(cpu) { struct pcpu_freelist_head *head = per_cpu_ptr(s->freelist, cpu); raw_spin_lock_init(&head->lock); head->first = NULL; } raw_spin_lock_init(&s->extralist.lock); s->extralist.first = NULL; return 0; } void pcpu_freelist_destroy(struct pcpu_freelist *s) { free_percpu(s->freelist); } static inline void pcpu_freelist_push_node(struct pcpu_freelist_head *head, struct pcpu_freelist_node *node) { node->next = head->first; WRITE_ONCE(head->first, node); } static inline void ___pcpu_freelist_push(struct pcpu_freelist_head *head, struct pcpu_freelist_node *node) { raw_spin_lock(&head->lock); pcpu_freelist_push_node(head, node); raw_spin_unlock(&head->lock); } static inline bool pcpu_freelist_try_push_extra(struct pcpu_freelist *s, struct pcpu_freelist_node *node) { if (!raw_spin_trylock(&s->extralist.lock)) return false; pcpu_freelist_push_node(&s->extralist, node); raw_spin_unlock(&s->extralist.lock); return true; } static inline void ___pcpu_freelist_push_nmi(struct pcpu_freelist *s, struct pcpu_freelist_node *node) { int cpu, orig_cpu; orig_cpu = raw_smp_processor_id(); while (1) { for_each_cpu_wrap(cpu, cpu_possible_mask, orig_cpu) { struct pcpu_freelist_head *head; head = per_cpu_ptr(s->freelist, cpu); if (raw_spin_trylock(&head->lock)) { pcpu_freelist_push_node(head, node); raw_spin_unlock(&head->lock); return; } } /* cannot lock any per cpu lock, try extralist */ if (pcpu_freelist_try_push_extra(s, node)) return; } } void __pcpu_freelist_push(struct pcpu_freelist *s, struct pcpu_freelist_node *node) { if (in_nmi()) ___pcpu_freelist_push_nmi(s, node); else ___pcpu_freelist_push(this_cpu_ptr(s->freelist), node); } void pcpu_freelist_push(struct pcpu_freelist *s, struct pcpu_freelist_node *node) { unsigned long flags; local_irq_save(flags); __pcpu_freelist_push(s, node); local_irq_restore(flags); } void pcpu_freelist_populate(struct pcpu_freelist *s, void *buf, u32 elem_size, u32 nr_elems) { struct pcpu_freelist_head *head; unsigned int cpu, cpu_idx, i, j, n, m; n = nr_elems / num_possible_cpus(); m = nr_elems % num_possible_cpus(); cpu_idx = 0; for_each_possible_cpu(cpu) { head = per_cpu_ptr(s->freelist, cpu); j = n + (cpu_idx < m ? 1 : 0); for (i = 0; i < j; i++) { /* No locking required as this is not visible yet. */ pcpu_freelist_push_node(head, buf); buf += elem_size; } cpu_idx++; } } static struct pcpu_freelist_node *___pcpu_freelist_pop(struct pcpu_freelist *s) { struct pcpu_freelist_head *head; struct pcpu_freelist_node *node; int cpu; for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) { head = per_cpu_ptr(s->freelist, cpu); if (!READ_ONCE(head->first)) continue; raw_spin_lock(&head->lock); node = head->first; if (node) { WRITE_ONCE(head->first, node->next); raw_spin_unlock(&head->lock); return node; } raw_spin_unlock(&head->lock); } /* per cpu lists are all empty, try extralist */ if (!READ_ONCE(s->extralist.first)) return NULL; raw_spin_lock(&s->extralist.lock); node = s->extralist.first; if (node) WRITE_ONCE(s->extralist.first, node->next); raw_spin_unlock(&s->extralist.lock); return node; } static struct pcpu_freelist_node * ___pcpu_freelist_pop_nmi(struct pcpu_freelist *s) { struct pcpu_freelist_head *head; struct pcpu_freelist_node *node; int cpu; for_each_cpu_wrap(cpu, cpu_possible_mask, raw_smp_processor_id()) { head = per_cpu_ptr(s->freelist, cpu); if (!READ_ONCE(head->first)) continue; if (raw_spin_trylock(&head->lock)) { node = head->first; if (node) { WRITE_ONCE(head->first, node->next); raw_spin_unlock(&head->lock); return node; } raw_spin_unlock(&head->lock); } } /* cannot pop from per cpu lists, try extralist */ if (!READ_ONCE(s->extralist.first) || !raw_spin_trylock(&s->extralist.lock)) return NULL; node = s->extralist.first; if (node) WRITE_ONCE(s->extralist.first, node->next); raw_spin_unlock(&s->extralist.lock); return node; } struct pcpu_freelist_node *__pcpu_freelist_pop(struct pcpu_freelist *s) { if (in_nmi()) return ___pcpu_freelist_pop_nmi(s); return ___pcpu_freelist_pop(s); } struct pcpu_freelist_node *pcpu_freelist_pop(struct pcpu_freelist *s) { struct pcpu_freelist_node *ret; unsigned long flags; local_irq_save(flags); ret = __pcpu_freelist_pop(s); local_irq_restore(flags); return ret; } |
| 1 1 1 1 16 10 5 1 1 15 13 2 12 3 12 2 1 9 4 9 6 3 5 16 2 2 10 5 3 3 5 4 2 2 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 | // SPDX-License-Identifier: GPL-2.0 #include <linux/types.h> #include <net/ip.h> #include <net/tcp.h> #include <net/netlink.h> #include <net/netfilter/nf_tables.h> #include <net/netfilter/nf_conntrack.h> #include <net/netfilter/nf_conntrack_synproxy.h> #include <net/netfilter/nf_synproxy.h> #include <linux/netfilter/nf_tables.h> #include <linux/netfilter/nf_synproxy.h> struct nft_synproxy { struct nf_synproxy_info info; }; static const struct nla_policy nft_synproxy_policy[NFTA_SYNPROXY_MAX + 1] = { [NFTA_SYNPROXY_MSS] = { .type = NLA_U16 }, [NFTA_SYNPROXY_WSCALE] = { .type = NLA_U8 }, [NFTA_SYNPROXY_FLAGS] = { .type = NLA_U32 }, }; static void nft_synproxy_tcp_options(struct synproxy_options *opts, const struct tcphdr *tcp, struct synproxy_net *snet, struct nf_synproxy_info *info, const struct nft_synproxy *priv) { this_cpu_inc(snet->stats->syn_received); if (tcp->ece && tcp->cwr) opts->options |= NF_SYNPROXY_OPT_ECN; opts->options &= priv->info.options; opts->mss_encode = opts->mss_option; opts->mss_option = info->mss; if (opts->options & NF_SYNPROXY_OPT_TIMESTAMP) synproxy_init_timestamp_cookie(info, opts); else opts->options &= ~(NF_SYNPROXY_OPT_WSCALE | NF_SYNPROXY_OPT_SACK_PERM | NF_SYNPROXY_OPT_ECN); } static void nft_synproxy_eval_v4(const struct nft_synproxy *priv, struct nft_regs *regs, const struct nft_pktinfo *pkt, const struct tcphdr *tcp, struct tcphdr *_tcph, struct synproxy_options *opts) { struct nf_synproxy_info info = priv->info; struct net *net = nft_net(pkt); struct synproxy_net *snet = synproxy_pernet(net); struct sk_buff *skb = pkt->skb; if (tcp->syn) { /* Initial SYN from client */ nft_synproxy_tcp_options(opts, tcp, snet, &info, priv); synproxy_send_client_synack(net, skb, tcp, opts); consume_skb(skb); regs->verdict.code = NF_STOLEN; } else if (tcp->ack) { /* ACK from client */ if (synproxy_recv_client_ack(net, skb, tcp, opts, ntohl(tcp->seq))) { consume_skb(skb); regs->verdict.code = NF_STOLEN; } else { regs->verdict.code = NF_DROP; } } } #if IS_ENABLED(CONFIG_NF_TABLES_IPV6) static void nft_synproxy_eval_v6(const struct nft_synproxy *priv, struct nft_regs *regs, const struct nft_pktinfo *pkt, const struct tcphdr *tcp, struct tcphdr *_tcph, struct synproxy_options *opts) { struct nf_synproxy_info info = priv->info; struct net *net = nft_net(pkt); struct synproxy_net *snet = synproxy_pernet(net); struct sk_buff *skb = pkt->skb; if (tcp->syn) { /* Initial SYN from client */ nft_synproxy_tcp_options(opts, tcp, snet, &info, priv); synproxy_send_client_synack_ipv6(net, skb, tcp, opts); consume_skb(skb); regs->verdict.code = NF_STOLEN; } else if (tcp->ack) { /* ACK from client */ if (synproxy_recv_client_ack_ipv6(net, skb, tcp, opts, ntohl(tcp->seq))) { consume_skb(skb); regs->verdict.code = NF_STOLEN; } else { regs->verdict.code = NF_DROP; } } } #endif /* CONFIG_NF_TABLES_IPV6*/ static void nft_synproxy_do_eval(const struct nft_synproxy *priv, struct nft_regs *regs, const struct nft_pktinfo *pkt) { struct synproxy_options opts = {}; struct sk_buff *skb = pkt->skb; int thoff = nft_thoff(pkt); const struct tcphdr *tcp; struct tcphdr _tcph; if (pkt->tprot != IPPROTO_TCP) { regs->verdict.code = NFT_BREAK; return; } if (nf_ip_checksum(skb, nft_hook(pkt), thoff, IPPROTO_TCP)) { regs->verdict.code = NF_DROP; return; } tcp = skb_header_pointer(skb, thoff, sizeof(struct tcphdr), &_tcph); if (!tcp) { regs->verdict.code = NF_DROP; return; } if (!synproxy_parse_options(skb, thoff, tcp, &opts)) { regs->verdict.code = NF_DROP; return; } switch (skb->protocol) { case htons(ETH_P_IP): nft_synproxy_eval_v4(priv, regs, pkt, tcp, &_tcph, &opts); return; #if IS_ENABLED(CONFIG_NF_TABLES_IPV6) case htons(ETH_P_IPV6): nft_synproxy_eval_v6(priv, regs, pkt, tcp, &_tcph, &opts); return; #endif } regs->verdict.code = NFT_BREAK; } static int nft_synproxy_do_init(const struct nft_ctx *ctx, const struct nlattr * const tb[], struct nft_synproxy *priv) { struct synproxy_net *snet = synproxy_pernet(ctx->net); u32 flags; int err; if (tb[NFTA_SYNPROXY_MSS]) priv->info.mss = ntohs(nla_get_be16(tb[NFTA_SYNPROXY_MSS])); if (tb[NFTA_SYNPROXY_WSCALE]) priv->info.wscale = nla_get_u8(tb[NFTA_SYNPROXY_WSCALE]); if (tb[NFTA_SYNPROXY_FLAGS]) { flags = ntohl(nla_get_be32(tb[NFTA_SYNPROXY_FLAGS])); if (flags & ~NF_SYNPROXY_OPT_MASK) return -EOPNOTSUPP; priv->info.options = flags; } err = nf_ct_netns_get(ctx->net, ctx->family); if (err) return err; switch (ctx->family) { case NFPROTO_IPV4: err = nf_synproxy_ipv4_init(snet, ctx->net); if (err) goto nf_ct_failure; break; #if IS_ENABLED(CONFIG_NF_TABLES_IPV6) case NFPROTO_IPV6: err = nf_synproxy_ipv6_init(snet, ctx->net); if (err) goto nf_ct_failure; break; #endif case NFPROTO_INET: err = nf_synproxy_ipv4_init(snet, ctx->net); if (err) goto nf_ct_failure; err = nf_synproxy_ipv6_init(snet, ctx->net); if (err) { nf_synproxy_ipv4_fini(snet, ctx->net); goto nf_ct_failure; } break; } return 0; nf_ct_failure: nf_ct_netns_put(ctx->net, ctx->family); return err; } static void nft_synproxy_do_destroy(const struct nft_ctx *ctx) { struct synproxy_net *snet = synproxy_pernet(ctx->net); switch (ctx->family) { case NFPROTO_IPV4: nf_synproxy_ipv4_fini(snet, ctx->net); break; #if IS_ENABLED(CONFIG_NF_TABLES_IPV6) case NFPROTO_IPV6: nf_synproxy_ipv6_fini(snet, ctx->net); break; #endif case NFPROTO_INET: nf_synproxy_ipv4_fini(snet, ctx->net); nf_synproxy_ipv6_fini(snet, ctx->net); break; } nf_ct_netns_put(ctx->net, ctx->family); } static int nft_synproxy_do_dump(struct sk_buff *skb, struct nft_synproxy *priv) { if (nla_put_be16(skb, NFTA_SYNPROXY_MSS, htons(priv->info.mss)) || nla_put_u8(skb, NFTA_SYNPROXY_WSCALE, priv->info.wscale) || nla_put_be32(skb, NFTA_SYNPROXY_FLAGS, htonl(priv->info.options))) goto nla_put_failure; return 0; nla_put_failure: return -1; } static void nft_synproxy_eval(const struct nft_expr *expr, struct nft_regs *regs, const struct nft_pktinfo *pkt) { const struct nft_synproxy *priv = nft_expr_priv(expr); nft_synproxy_do_eval(priv, regs, pkt); } static int nft_synproxy_validate(const struct nft_ctx *ctx, const struct nft_expr *expr) { if (ctx->family != NFPROTO_IPV4 && ctx->family != NFPROTO_IPV6 && ctx->family != NFPROTO_INET) return -EOPNOTSUPP; return nft_chain_validate_hooks(ctx->chain, (1 << NF_INET_LOCAL_IN) | (1 << NF_INET_FORWARD)); } static int nft_synproxy_init(const struct nft_ctx *ctx, const struct nft_expr *expr, const struct nlattr * const tb[]) { struct nft_synproxy *priv = nft_expr_priv(expr); return nft_synproxy_do_init(ctx, tb, priv); } static void nft_synproxy_destroy(const struct nft_ctx *ctx, const struct nft_expr *expr) { nft_synproxy_do_destroy(ctx); } static int nft_synproxy_dump(struct sk_buff *skb, const struct nft_expr *expr, bool reset) { struct nft_synproxy *priv = nft_expr_priv(expr); return nft_synproxy_do_dump(skb, priv); } static struct nft_expr_type nft_synproxy_type; static const struct nft_expr_ops nft_synproxy_ops = { .eval = nft_synproxy_eval, .size = NFT_EXPR_SIZE(sizeof(struct nft_synproxy)), .init = nft_synproxy_init, .destroy = nft_synproxy_destroy, .dump = nft_synproxy_dump, .type = &nft_synproxy_type, .validate = nft_synproxy_validate, .reduce = NFT_REDUCE_READONLY, }; static struct nft_expr_type nft_synproxy_type __read_mostly = { .ops = &nft_synproxy_ops, .name = "synproxy", .owner = THIS_MODULE, .policy = nft_synproxy_policy, .maxattr = NFTA_SYNPROXY_MAX, }; static int nft_synproxy_obj_init(const struct nft_ctx *ctx, const struct nlattr * const tb[], struct nft_object *obj) { struct nft_synproxy *priv = nft_obj_data(obj); return nft_synproxy_do_init(ctx, tb, priv); } static void nft_synproxy_obj_destroy(const struct nft_ctx *ctx, struct nft_object *obj) { nft_synproxy_do_destroy(ctx); } static int nft_synproxy_obj_dump(struct sk_buff *skb, struct nft_object *obj, bool reset) { struct nft_synproxy *priv = nft_obj_data(obj); return nft_synproxy_do_dump(skb, priv); } static void nft_synproxy_obj_eval(struct nft_object *obj, struct nft_regs *regs, const struct nft_pktinfo *pkt) { const struct nft_synproxy *priv = nft_obj_data(obj); nft_synproxy_do_eval(priv, regs, pkt); } static void nft_synproxy_obj_update(struct nft_object *obj, struct nft_object *newobj) { struct nft_synproxy *newpriv = nft_obj_data(newobj); struct nft_synproxy *priv = nft_obj_data(obj); priv->info = newpriv->info; } static struct nft_object_type nft_synproxy_obj_type; static const struct nft_object_ops nft_synproxy_obj_ops = { .type = &nft_synproxy_obj_type, .size = sizeof(struct nft_synproxy), .init = nft_synproxy_obj_init, .destroy = nft_synproxy_obj_destroy, .dump = nft_synproxy_obj_dump, .eval = nft_synproxy_obj_eval, .update = nft_synproxy_obj_update, }; static struct nft_object_type nft_synproxy_obj_type __read_mostly = { .type = NFT_OBJECT_SYNPROXY, .ops = &nft_synproxy_obj_ops, .maxattr = NFTA_SYNPROXY_MAX, .policy = nft_synproxy_policy, .owner = THIS_MODULE, }; static int __init nft_synproxy_module_init(void) { int err; err = nft_register_obj(&nft_synproxy_obj_type); if (err < 0) return err; err = nft_register_expr(&nft_synproxy_type); if (err < 0) goto err; return 0; err: nft_unregister_obj(&nft_synproxy_obj_type); return err; } static void __exit nft_synproxy_module_exit(void) { nft_unregister_expr(&nft_synproxy_type); nft_unregister_obj(&nft_synproxy_obj_type); } module_init(nft_synproxy_module_init); module_exit(nft_synproxy_module_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Fernando Fernandez <ffmancera@riseup.net>"); MODULE_ALIAS_NFT_EXPR("synproxy"); MODULE_ALIAS_NFT_OBJ(NFT_OBJECT_SYNPROXY); MODULE_DESCRIPTION("nftables SYNPROXY expression support"); |
| 14 3 4 2 2 15 1 4 10 45 4 30 5 10 1 42 3 21 2 19 38 2 3 3 1 10 1 4 5 6 6 5 2 2 1 1 12 2 1 1 1 7 3 3 10 4 1 13 13 13 13 13 13 22 2 32 1 32 50 2 35 17 6 6 6 1 17 17 3 11 9 6 5 5 9 9 4 7 7 4 1 2 1 115 71 51 3 46 3 5 4 3 4 3 3 4 4 3 3 3 1 2 1 2 1 1 1 1 1 1 1 1 1 1 2 3 4 12 115 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 | // SPDX-License-Identifier: GPL-2.0-or-later /* * User level driver support for input subsystem * * Heavily based on evdev.c by Vojtech Pavlik * * Author: Aristeu Sergio Rozanski Filho <aris@cathedrallabs.org> * * Changes/Revisions: * 0.4 01/09/2014 (Benjamin Tissoires <benjamin.tissoires@redhat.com>) * - add UI_GET_SYSNAME ioctl * 0.3 09/04/2006 (Anssi Hannula <anssi.hannula@gmail.com>) * - updated ff support for the changes in kernel interface * - added MODULE_VERSION * 0.2 16/10/2004 (Micah Dowty <micah@navi.cx>) * - added force feedback support * - added UI_SET_PHYS * 0.1 20/06/2002 * - first public version */ #include <uapi/linux/uinput.h> #include <linux/poll.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/module.h> #include <linux/init.h> #include <linux/fs.h> #include <linux/miscdevice.h> #include <linux/overflow.h> #include <linux/input/mt.h> #include "../input-compat.h" #define UINPUT_NAME "uinput" #define UINPUT_BUFFER_SIZE 16 #define UINPUT_NUM_REQUESTS 16 #define UINPUT_TIMESTAMP_ALLOWED_OFFSET_SECS 10 enum uinput_state { UIST_NEW_DEVICE, UIST_SETUP_COMPLETE, UIST_CREATED }; struct uinput_request { unsigned int id; unsigned int code; /* UI_FF_UPLOAD, UI_FF_ERASE */ int retval; struct completion done; union { unsigned int effect_id; struct { struct ff_effect *effect; struct ff_effect *old; } upload; } u; }; struct uinput_device { struct input_dev *dev; struct mutex mutex; enum uinput_state state; wait_queue_head_t waitq; unsigned char ready; unsigned char head; unsigned char tail; struct input_event buff[UINPUT_BUFFER_SIZE]; unsigned int ff_effects_max; struct uinput_request *requests[UINPUT_NUM_REQUESTS]; wait_queue_head_t requests_waitq; spinlock_t requests_lock; }; static int uinput_dev_event(struct input_dev *dev, unsigned int type, unsigned int code, int value) { struct uinput_device *udev = input_get_drvdata(dev); struct timespec64 ts; ktime_get_ts64(&ts); udev->buff[udev->head] = (struct input_event) { .input_event_sec = ts.tv_sec, .input_event_usec = ts.tv_nsec / NSEC_PER_USEC, .type = type, .code = code, .value = value, }; udev->head = (udev->head + 1) % UINPUT_BUFFER_SIZE; wake_up_interruptible(&udev->waitq); return 0; } /* Atomically allocate an ID for the given request. Returns 0 on success. */ static bool uinput_request_alloc_id(struct uinput_device *udev, struct uinput_request *request) { unsigned int id; bool reserved = false; spin_lock(&udev->requests_lock); for (id = 0; id < UINPUT_NUM_REQUESTS; id++) { if (!udev->requests[id]) { request->id = id; udev->requests[id] = request; reserved = true; break; } } spin_unlock(&udev->requests_lock); return reserved; } static struct uinput_request *uinput_request_find(struct uinput_device *udev, unsigned int id) { /* Find an input request, by ID. Returns NULL if the ID isn't valid. */ if (id >= UINPUT_NUM_REQUESTS) return NULL; return udev->requests[id]; } static int uinput_request_reserve_slot(struct uinput_device *udev, struct uinput_request *request) { /* Allocate slot. If none are available right away, wait. */ return wait_event_interruptible(udev->requests_waitq, uinput_request_alloc_id(udev, request)); } static void uinput_request_release_slot(struct uinput_device *udev, unsigned int id) { /* Mark slot as available */ spin_lock(&udev->requests_lock); udev->requests[id] = NULL; spin_unlock(&udev->requests_lock); wake_up(&udev->requests_waitq); } static int uinput_request_send(struct uinput_device *udev, struct uinput_request *request) { int retval; retval = mutex_lock_interruptible(&udev->mutex); if (retval) return retval; if (udev->state != UIST_CREATED) { retval = -ENODEV; goto out; } init_completion(&request->done); /* * Tell our userspace application about this new request * by queueing an input event. */ uinput_dev_event(udev->dev, EV_UINPUT, request->code, request->id); out: mutex_unlock(&udev->mutex); return retval; } static int uinput_request_submit(struct uinput_device *udev, struct uinput_request *request) { int retval; retval = uinput_request_reserve_slot(udev, request); if (retval) return retval; retval = uinput_request_send(udev, request); if (retval) goto out; if (!wait_for_completion_timeout(&request->done, 30 * HZ)) { retval = -ETIMEDOUT; goto out; } retval = request->retval; out: uinput_request_release_slot(udev, request->id); return retval; } /* * Fail all outstanding requests so handlers don't wait for the userspace * to finish processing them. */ static void uinput_flush_requests(struct uinput_device *udev) { struct uinput_request *request; int i; spin_lock(&udev->requests_lock); for (i = 0; i < UINPUT_NUM_REQUESTS; i++) { request = udev->requests[i]; if (request) { request->retval = -ENODEV; complete(&request->done); } } spin_unlock(&udev->requests_lock); } static void uinput_dev_set_gain(struct input_dev *dev, u16 gain) { uinput_dev_event(dev, EV_FF, FF_GAIN, gain); } static void uinput_dev_set_autocenter(struct input_dev *dev, u16 magnitude) { uinput_dev_event(dev, EV_FF, FF_AUTOCENTER, magnitude); } static int uinput_dev_playback(struct input_dev *dev, int effect_id, int value) { return uinput_dev_event(dev, EV_FF, effect_id, value); } static int uinput_dev_upload_effect(struct input_dev *dev, struct ff_effect *effect, struct ff_effect *old) { struct uinput_device *udev = input_get_drvdata(dev); struct uinput_request request; /* * uinput driver does not currently support periodic effects with * custom waveform since it does not have a way to pass buffer of * samples (custom_data) to userspace. If ever there is a device * supporting custom waveforms we would need to define an additional * ioctl (UI_UPLOAD_SAMPLES) but for now we just bail out. */ if (effect->type == FF_PERIODIC && effect->u.periodic.waveform == FF_CUSTOM) return -EINVAL; request.code = UI_FF_UPLOAD; request.u.upload.effect = effect; request.u.upload.old = old; return uinput_request_submit(udev, &request); } static int uinput_dev_erase_effect(struct input_dev *dev, int effect_id) { struct uinput_device *udev = input_get_drvdata(dev); struct uinput_request request; if (!test_bit(EV_FF, dev->evbit)) return -ENOSYS; request.code = UI_FF_ERASE; request.u.effect_id = effect_id; return uinput_request_submit(udev, &request); } static int uinput_dev_flush(struct input_dev *dev, struct file *file) { /* * If we are called with file == NULL that means we are tearing * down the device, and therefore we can not handle FF erase * requests: either we are handling UI_DEV_DESTROY (and holding * the udev->mutex), or the file descriptor is closed and there is * nobody on the other side anymore. */ return file ? input_ff_flush(dev, file) : 0; } static void uinput_destroy_device(struct uinput_device *udev) { const char *name, *phys; struct input_dev *dev = udev->dev; enum uinput_state old_state = udev->state; udev->state = UIST_NEW_DEVICE; if (dev) { name = dev->name; phys = dev->phys; if (old_state == UIST_CREATED) { uinput_flush_requests(udev); input_unregister_device(dev); } else { input_free_device(dev); } kfree(name); kfree(phys); udev->dev = NULL; } } static int uinput_create_device(struct uinput_device *udev) { struct input_dev *dev = udev->dev; int error, nslot; if (udev->state != UIST_SETUP_COMPLETE) { printk(KERN_DEBUG "%s: write device info first\n", UINPUT_NAME); return -EINVAL; } if (test_bit(EV_ABS, dev->evbit)) { input_alloc_absinfo(dev); if (!dev->absinfo) { error = -EINVAL; goto fail1; } if (test_bit(ABS_MT_SLOT, dev->absbit)) { nslot = input_abs_get_max(dev, ABS_MT_SLOT) + 1; error = input_mt_init_slots(dev, nslot, 0); if (error) goto fail1; } else if (test_bit(ABS_MT_POSITION_X, dev->absbit)) { input_set_events_per_packet(dev, 60); } } if (test_bit(EV_FF, dev->evbit) && !udev->ff_effects_max) { printk(KERN_DEBUG "%s: ff_effects_max should be non-zero when FF_BIT is set\n", UINPUT_NAME); error = -EINVAL; goto fail1; } if (udev->ff_effects_max) { error = input_ff_create(dev, udev->ff_effects_max); if (error) goto fail1; dev->ff->upload = uinput_dev_upload_effect; dev->ff->erase = uinput_dev_erase_effect; dev->ff->playback = uinput_dev_playback; dev->ff->set_gain = uinput_dev_set_gain; dev->ff->set_autocenter = uinput_dev_set_autocenter; /* * The standard input_ff_flush() implementation does * not quite work for uinput as we can't reasonably * handle FF requests during device teardown. */ dev->flush = uinput_dev_flush; } dev->event = uinput_dev_event; input_set_drvdata(udev->dev, udev); error = input_register_device(udev->dev); if (error) goto fail2; udev->state = UIST_CREATED; return 0; fail2: input_ff_destroy(dev); fail1: uinput_destroy_device(udev); return error; } static int uinput_open(struct inode *inode, struct file *file) { struct uinput_device *newdev; newdev = kzalloc(sizeof(*newdev), GFP_KERNEL); if (!newdev) return -ENOMEM; mutex_init(&newdev->mutex); spin_lock_init(&newdev->requests_lock); init_waitqueue_head(&newdev->requests_waitq); init_waitqueue_head(&newdev->waitq); newdev->state = UIST_NEW_DEVICE; file->private_data = newdev; stream_open(inode, file); return 0; } static int uinput_validate_absinfo(struct input_dev *dev, unsigned int code, const struct input_absinfo *abs) { int min, max, range; min = abs->minimum; max = abs->maximum; if ((min != 0 || max != 0) && max < min) { printk(KERN_DEBUG "%s: invalid abs[%02x] min:%d max:%d\n", UINPUT_NAME, code, min, max); return -EINVAL; } if (!check_sub_overflow(max, min, &range) && abs->flat > range) { printk(KERN_DEBUG "%s: abs_flat #%02x out of range: %d (min:%d/max:%d)\n", UINPUT_NAME, code, abs->flat, min, max); return -EINVAL; } /* * Limit number of contacts to a reasonable value (100). This * ensures that we need less than 2 pages for struct input_mt * (we are not using in-kernel slot assignment so not going to * allocate memory for the "red" table), and we should have no * trouble getting this much memory. */ if (code == ABS_MT_SLOT && max > 99) { printk(KERN_DEBUG "%s: unreasonably large number of slots requested: %d\n", UINPUT_NAME, max); return -EINVAL; } return 0; } static int uinput_validate_absbits(struct input_dev *dev) { unsigned int cnt; int error; if (!test_bit(EV_ABS, dev->evbit)) return 0; /* * Check if absmin/absmax/absfuzz/absflat are sane. */ for_each_set_bit(cnt, dev->absbit, ABS_CNT) { if (!dev->absinfo) return -EINVAL; error = uinput_validate_absinfo(dev, cnt, &dev->absinfo[cnt]); if (error) return error; } return 0; } static int uinput_dev_setup(struct uinput_device *udev, struct uinput_setup __user *arg) { struct uinput_setup setup; struct input_dev *dev; if (udev->state == UIST_CREATED) return -EINVAL; if (copy_from_user(&setup, arg, sizeof(setup))) return -EFAULT; if (!setup.name[0]) return -EINVAL; dev = udev->dev; dev->id = setup.id; udev->ff_effects_max = setup.ff_effects_max; kfree(dev->name); dev->name = kstrndup(setup.name, UINPUT_MAX_NAME_SIZE, GFP_KERNEL); if (!dev->name) return -ENOMEM; udev->state = UIST_SETUP_COMPLETE; return 0; } static int uinput_abs_setup(struct uinput_device *udev, struct uinput_setup __user *arg, size_t size) { struct uinput_abs_setup setup = {}; struct input_dev *dev; int error; if (size > sizeof(setup)) return -E2BIG; if (udev->state == UIST_CREATED) return -EINVAL; if (copy_from_user(&setup, arg, size)) return -EFAULT; if (setup.code > ABS_MAX) return -ERANGE; dev = udev->dev; error = uinput_validate_absinfo(dev, setup.code, &setup.absinfo); if (error) return error; input_alloc_absinfo(dev); if (!dev->absinfo) return -ENOMEM; set_bit(setup.code, dev->absbit); dev->absinfo[setup.code] = setup.absinfo; return 0; } /* legacy setup via write() */ static int uinput_setup_device_legacy(struct uinput_device *udev, const char __user *buffer, size_t count) { struct uinput_user_dev *user_dev; struct input_dev *dev; int i; int retval; if (count != sizeof(struct uinput_user_dev)) return -EINVAL; if (!udev->dev) { udev->dev = input_allocate_device(); if (!udev->dev) return -ENOMEM; } dev = udev->dev; user_dev = memdup_user(buffer, sizeof(struct uinput_user_dev)); if (IS_ERR(user_dev)) return PTR_ERR(user_dev); udev->ff_effects_max = user_dev->ff_effects_max; /* Ensure name is filled in */ if (!user_dev->name[0]) { retval = -EINVAL; goto exit; } kfree(dev->name); dev->name = kstrndup(user_dev->name, UINPUT_MAX_NAME_SIZE, GFP_KERNEL); if (!dev->name) { retval = -ENOMEM; goto exit; } dev->id.bustype = user_dev->id.bustype; dev->id.vendor = user_dev->id.vendor; dev->id.product = user_dev->id.product; dev->id.version = user_dev->id.version; for (i = 0; i < ABS_CNT; i++) { input_abs_set_max(dev, i, user_dev->absmax[i]); input_abs_set_min(dev, i, user_dev->absmin[i]); input_abs_set_fuzz(dev, i, user_dev->absfuzz[i]); input_abs_set_flat(dev, i, user_dev->absflat[i]); } retval = uinput_validate_absbits(dev); if (retval < 0) goto exit; udev->state = UIST_SETUP_COMPLETE; retval = count; exit: kfree(user_dev); return retval; } /* * Returns true if the given timestamp is valid (i.e., if all the following * conditions are satisfied), false otherwise. * 1) given timestamp is positive * 2) it's within the allowed offset before the current time * 3) it's not in the future */ static bool is_valid_timestamp(const ktime_t timestamp) { ktime_t zero_time; ktime_t current_time; ktime_t min_time; ktime_t offset; zero_time = ktime_set(0, 0); if (ktime_compare(zero_time, timestamp) >= 0) return false; current_time = ktime_get(); offset = ktime_set(UINPUT_TIMESTAMP_ALLOWED_OFFSET_SECS, 0); min_time = ktime_sub(current_time, offset); if (ktime_after(min_time, timestamp) || ktime_after(timestamp, current_time)) return false; return true; } static ssize_t uinput_inject_events(struct uinput_device *udev, const char __user *buffer, size_t count) { struct input_event ev; size_t bytes = 0; ktime_t timestamp; if (count != 0 && count < input_event_size()) return -EINVAL; while (bytes + input_event_size() <= count) { /* * Note that even if some events were fetched successfully * we are still going to return EFAULT instead of partial * count to let userspace know that it got it's buffers * all wrong. */ if (input_event_from_user(buffer + bytes, &ev)) return -EFAULT; timestamp = ktime_set(ev.input_event_sec, ev.input_event_usec * NSEC_PER_USEC); if (is_valid_timestamp(timestamp)) input_set_timestamp(udev->dev, timestamp); input_event(udev->dev, ev.type, ev.code, ev.value); bytes += input_event_size(); cond_resched(); } return bytes; } static ssize_t uinput_write(struct file *file, const char __user *buffer, size_t count, loff_t *ppos) { struct uinput_device *udev = file->private_data; int retval; if (count == 0) return 0; retval = mutex_lock_interruptible(&udev->mutex); if (retval) return retval; retval = udev->state == UIST_CREATED ? uinput_inject_events(udev, buffer, count) : uinput_setup_device_legacy(udev, buffer, count); mutex_unlock(&udev->mutex); return retval; } static bool uinput_fetch_next_event(struct uinput_device *udev, struct input_event *event) { bool have_event; spin_lock_irq(&udev->dev->event_lock); have_event = udev->head != udev->tail; if (have_event) { *event = udev->buff[udev->tail]; udev->tail = (udev->tail + 1) % UINPUT_BUFFER_SIZE; } spin_unlock_irq(&udev->dev->event_lock); return have_event; } static ssize_t uinput_events_to_user(struct uinput_device *udev, char __user *buffer, size_t count) { struct input_event event; size_t read = 0; while (read + input_event_size() <= count && uinput_fetch_next_event(udev, &event)) { if (input_event_to_user(buffer + read, &event)) return -EFAULT; read += input_event_size(); } return read; } static ssize_t uinput_read(struct file *file, char __user *buffer, size_t count, loff_t *ppos) { struct uinput_device *udev = file->private_data; ssize_t retval; if (count != 0 && count < input_event_size()) return -EINVAL; do { retval = mutex_lock_interruptible(&udev->mutex); if (retval) return retval; if (udev->state != UIST_CREATED) retval = -ENODEV; else if (udev->head == udev->tail && (file->f_flags & O_NONBLOCK)) retval = -EAGAIN; else retval = uinput_events_to_user(udev, buffer, count); mutex_unlock(&udev->mutex); if (retval || count == 0) break; if (!(file->f_flags & O_NONBLOCK)) retval = wait_event_interruptible(udev->waitq, udev->head != udev->tail || udev->state != UIST_CREATED); } while (retval == 0); return retval; } static __poll_t uinput_poll(struct file *file, poll_table *wait) { struct uinput_device *udev = file->private_data; __poll_t mask = EPOLLOUT | EPOLLWRNORM; /* uinput is always writable */ poll_wait(file, &udev->waitq, wait); if (udev->head != udev->tail) mask |= EPOLLIN | EPOLLRDNORM; return mask; } static int uinput_release(struct inode *inode, struct file *file) { struct uinput_device *udev = file->private_data; uinput_destroy_device(udev); kfree(udev); return 0; } #ifdef CONFIG_COMPAT struct uinput_ff_upload_compat { __u32 request_id; __s32 retval; struct ff_effect_compat effect; struct ff_effect_compat old; }; static int uinput_ff_upload_to_user(char __user *buffer, const struct uinput_ff_upload *ff_up) { if (in_compat_syscall()) { struct uinput_ff_upload_compat ff_up_compat; ff_up_compat.request_id = ff_up->request_id; ff_up_compat.retval = ff_up->retval; /* * It so happens that the pointer that gives us the trouble * is the last field in the structure. Since we don't support * custom waveforms in uinput anyway we can just copy the whole * thing (to the compat size) and ignore the pointer. */ memcpy(&ff_up_compat.effect, &ff_up->effect, sizeof(struct ff_effect_compat)); memcpy(&ff_up_compat.old, &ff_up->old, sizeof(struct ff_effect_compat)); if (copy_to_user(buffer, &ff_up_compat, sizeof(struct uinput_ff_upload_compat))) return -EFAULT; } else { if (copy_to_user(buffer, ff_up, sizeof(struct uinput_ff_upload))) return -EFAULT; } return 0; } static int uinput_ff_upload_from_user(const char __user *buffer, struct uinput_ff_upload *ff_up) { if (in_compat_syscall()) { struct uinput_ff_upload_compat ff_up_compat; if (copy_from_user(&ff_up_compat, buffer, sizeof(struct uinput_ff_upload_compat))) return -EFAULT; ff_up->request_id = ff_up_compat.request_id; ff_up->retval = ff_up_compat.retval; memcpy(&ff_up->effect, &ff_up_compat.effect, sizeof(struct ff_effect_compat)); memcpy(&ff_up->old, &ff_up_compat.old, sizeof(struct ff_effect_compat)); } else { if (copy_from_user(ff_up, buffer, sizeof(struct uinput_ff_upload))) return -EFAULT; } return 0; } #else static int uinput_ff_upload_to_user(char __user *buffer, const struct uinput_ff_upload *ff_up) { if (copy_to_user(buffer, ff_up, sizeof(struct uinput_ff_upload))) return -EFAULT; return 0; } static int uinput_ff_upload_from_user(const char __user *buffer, struct uinput_ff_upload *ff_up) { if (copy_from_user(ff_up, buffer, sizeof(struct uinput_ff_upload))) return -EFAULT; return 0; } #endif #define uinput_set_bit(_arg, _bit, _max) \ ({ \ int __ret = 0; \ if (udev->state == UIST_CREATED) \ __ret = -EINVAL; \ else if ((_arg) > (_max)) \ __ret = -EINVAL; \ else set_bit((_arg), udev->dev->_bit); \ __ret; \ }) static int uinput_str_to_user(void __user *dest, const char *str, unsigned int maxlen) { char __user *p = dest; int len, ret; if (!str) return -ENOENT; if (maxlen == 0) return -EINVAL; len = strlen(str) + 1; if (len > maxlen) len = maxlen; ret = copy_to_user(p, str, len); if (ret) return -EFAULT; /* force terminating '\0' */ ret = put_user(0, p + len - 1); return ret ? -EFAULT : len; } static long uinput_ioctl_handler(struct file *file, unsigned int cmd, unsigned long arg, void __user *p) { int retval; struct uinput_device *udev = file->private_data; struct uinput_ff_upload ff_up; struct uinput_ff_erase ff_erase; struct uinput_request *req; char *phys; const char *name; unsigned int size; retval = mutex_lock_interruptible(&udev->mutex); if (retval) return retval; if (!udev->dev) { udev->dev = input_allocate_device(); if (!udev->dev) { retval = -ENOMEM; goto out; } } switch (cmd) { case UI_GET_VERSION: if (put_user(UINPUT_VERSION, (unsigned int __user *)p)) retval = -EFAULT; goto out; case UI_DEV_CREATE: retval = uinput_create_device(udev); goto out; case UI_DEV_DESTROY: uinput_destroy_device(udev); goto out; case UI_DEV_SETUP: retval = uinput_dev_setup(udev, p); goto out; /* UI_ABS_SETUP is handled in the variable size ioctls */ case UI_SET_EVBIT: retval = uinput_set_bit(arg, evbit, EV_MAX); goto out; case UI_SET_KEYBIT: retval = uinput_set_bit(arg, keybit, KEY_MAX); goto out; case UI_SET_RELBIT: retval = uinput_set_bit(arg, relbit, REL_MAX); goto out; case UI_SET_ABSBIT: retval = uinput_set_bit(arg, absbit, ABS_MAX); goto out; case UI_SET_MSCBIT: retval = uinput_set_bit(arg, mscbit, MSC_MAX); goto out; case UI_SET_LEDBIT: retval = uinput_set_bit(arg, ledbit, LED_MAX); goto out; case UI_SET_SNDBIT: retval = uinput_set_bit(arg, sndbit, SND_MAX); goto out; case UI_SET_FFBIT: retval = uinput_set_bit(arg, ffbit, FF_MAX); goto out; case UI_SET_SWBIT: retval = uinput_set_bit(arg, swbit, SW_MAX); goto out; case UI_SET_PROPBIT: retval = uinput_set_bit(arg, propbit, INPUT_PROP_MAX); goto out; case UI_SET_PHYS: if (udev->state == UIST_CREATED) { retval = -EINVAL; goto out; } phys = strndup_user(p, 1024); if (IS_ERR(phys)) { retval = PTR_ERR(phys); goto out; } kfree(udev->dev->phys); udev->dev->phys = phys; goto out; case UI_BEGIN_FF_UPLOAD: retval = uinput_ff_upload_from_user(p, &ff_up); if (retval) goto out; req = uinput_request_find(udev, ff_up.request_id); if (!req || req->code != UI_FF_UPLOAD || !req->u.upload.effect) { retval = -EINVAL; goto out; } ff_up.retval = 0; ff_up.effect = *req->u.upload.effect; if (req->u.upload.old) ff_up.old = *req->u.upload.old; else memset(&ff_up.old, 0, sizeof(struct ff_effect)); retval = uinput_ff_upload_to_user(p, &ff_up); goto out; case UI_BEGIN_FF_ERASE: if (copy_from_user(&ff_erase, p, sizeof(ff_erase))) { retval = -EFAULT; goto out; } req = uinput_request_find(udev, ff_erase.request_id); if (!req || req->code != UI_FF_ERASE) { retval = -EINVAL; goto out; } ff_erase.retval = 0; ff_erase.effect_id = req->u.effect_id; if (copy_to_user(p, &ff_erase, sizeof(ff_erase))) { retval = -EFAULT; goto out; } goto out; case UI_END_FF_UPLOAD: retval = uinput_ff_upload_from_user(p, &ff_up); if (retval) goto out; req = uinput_request_find(udev, ff_up.request_id); if (!req || req->code != UI_FF_UPLOAD || !req->u.upload.effect) { retval = -EINVAL; goto out; } req->retval = ff_up.retval; complete(&req->done); goto out; case UI_END_FF_ERASE: if (copy_from_user(&ff_erase, p, sizeof(ff_erase))) { retval = -EFAULT; goto out; } req = uinput_request_find(udev, ff_erase.request_id); if (!req || req->code != UI_FF_ERASE) { retval = -EINVAL; goto out; } req->retval = ff_erase.retval; complete(&req->done); goto out; } size = _IOC_SIZE(cmd); /* Now check variable-length commands */ switch (cmd & ~IOCSIZE_MASK) { case UI_GET_SYSNAME(0): if (udev->state != UIST_CREATED) { retval = -ENOENT; goto out; } name = dev_name(&udev->dev->dev); retval = uinput_str_to_user(p, name, size); goto out; case UI_ABS_SETUP & ~IOCSIZE_MASK: retval = uinput_abs_setup(udev, p, size); goto out; } retval = -EINVAL; out: mutex_unlock(&udev->mutex); return retval; } static long uinput_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { return uinput_ioctl_handler(file, cmd, arg, (void __user *)arg); } #ifdef CONFIG_COMPAT /* * These IOCTLs change their size and thus their numbers between * 32 and 64 bits. */ #define UI_SET_PHYS_COMPAT \ _IOW(UINPUT_IOCTL_BASE, 108, compat_uptr_t) #define UI_BEGIN_FF_UPLOAD_COMPAT \ _IOWR(UINPUT_IOCTL_BASE, 200, struct uinput_ff_upload_compat) #define UI_END_FF_UPLOAD_COMPAT \ _IOW(UINPUT_IOCTL_BASE, 201, struct uinput_ff_upload_compat) static long uinput_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { switch (cmd) { case UI_SET_PHYS_COMPAT: cmd = UI_SET_PHYS; break; case UI_BEGIN_FF_UPLOAD_COMPAT: cmd = UI_BEGIN_FF_UPLOAD; break; case UI_END_FF_UPLOAD_COMPAT: cmd = UI_END_FF_UPLOAD; break; } return uinput_ioctl_handler(file, cmd, arg, compat_ptr(arg)); } #endif static const struct file_operations uinput_fops = { .owner = THIS_MODULE, .open = uinput_open, .release = uinput_release, .read = uinput_read, .write = uinput_write, .poll = uinput_poll, .unlocked_ioctl = uinput_ioctl, #ifdef CONFIG_COMPAT .compat_ioctl = uinput_compat_ioctl, #endif }; static struct miscdevice uinput_misc = { .fops = &uinput_fops, .minor = UINPUT_MINOR, .name = UINPUT_NAME, }; module_misc_device(uinput_misc); MODULE_ALIAS_MISCDEV(UINPUT_MINOR); MODULE_ALIAS("devname:" UINPUT_NAME); MODULE_AUTHOR("Aristeu Sergio Rozanski Filho"); MODULE_DESCRIPTION("User level driver support for input subsystem"); MODULE_LICENSE("GPL"); |
| 1 4 5 5 4 1 796 655 13 11 288 40 23 12 133 77 27 24 31 19 29 29 9 24 9 78 189 258 7 9 211 19 202 307 43 41 25 197 304 223 44 4 4 4 4 2 1 1 1 1 19 1 18 4 4 1 2 2 5 1 4 1 4 3 3 3 1 148 1 148 180 16 145 159 1 94 2 60 3 2 12 149 37 17 4 2 1 1 4 3 1 1 1 1 1 221 16 134 5 9 59 14 16 6 6 36 12 7 3 4 6 6 2 2 1 1 3 3 6 15 10 7 6 6 10 1 9 2 2 15 1 14 1 1 41 13 5 23 30 30 51 52 29 29 73 18 5 50 30 24 33 31 18 18 20 13 7 22 11 9 9 4 4 5 5 2 2 87 86 25 24 40 40 15 15 14 22 4 17 25 4 11 11 11 10 10 1 12 10 12 12 14 10 3 2 10 5 10 13 11 2 2 4 4 4 3 12 5 17 13 4 13 6 6 11 1 1 1 1 19 4 1 3 11 13 1 1 2 2 48 23 2 1 3 1 41 1 4 2 2 3 2 1285 142 1286 1420 1388 44 1430 1426 3 16 1 743 693 3 1419 1406 1387 45 1679 134 4 9 12 2 17 26 3 70 2 4 1680 35 168 1476 1645 360 76 856 1677 1680 7 1193 508 1687 1292 1680 837 859 1 11 1579 836 918 1428 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Video capture interface for Linux version 2 * * A generic framework to process V4L2 ioctl commands. * * Authors: Alan Cox, <alan@lxorguk.ukuu.org.uk> (version 1) * Mauro Carvalho Chehab <mchehab@kernel.org> (version 2) */ #include <linux/compat.h> #include <linux/mm.h> #include <linux/module.h> #include <linux/slab.h> #include <linux/types.h> #include <linux/kernel.h> #include <linux/version.h> #include <linux/v4l2-subdev.h> #include <linux/videodev2.h> #include <media/media-device.h> /* for media_set_bus_info() */ #include <media/v4l2-common.h> #include <media/v4l2-ioctl.h> #include <media/v4l2-ctrls.h> #include <media/v4l2-fh.h> #include <media/v4l2-event.h> #include <media/v4l2-device.h> #include <media/videobuf2-v4l2.h> #include <media/v4l2-mc.h> #include <media/v4l2-mem2mem.h> #include <trace/events/v4l2.h> #define is_valid_ioctl(vfd, cmd) test_bit(_IOC_NR(cmd), (vfd)->valid_ioctls) struct std_descr { v4l2_std_id std; const char *descr; }; static const struct std_descr standards[] = { { V4L2_STD_NTSC, "NTSC" }, { V4L2_STD_NTSC_M, "NTSC-M" }, { V4L2_STD_NTSC_M_JP, "NTSC-M-JP" }, { V4L2_STD_NTSC_M_KR, "NTSC-M-KR" }, { V4L2_STD_NTSC_443, "NTSC-443" }, { V4L2_STD_PAL, "PAL" }, { V4L2_STD_PAL_BG, "PAL-BG" }, { V4L2_STD_PAL_B, "PAL-B" }, { V4L2_STD_PAL_B1, "PAL-B1" }, { V4L2_STD_PAL_G, "PAL-G" }, { V4L2_STD_PAL_H, "PAL-H" }, { V4L2_STD_PAL_I, "PAL-I" }, { V4L2_STD_PAL_DK, "PAL-DK" }, { V4L2_STD_PAL_D, "PAL-D" }, { V4L2_STD_PAL_D1, "PAL-D1" }, { V4L2_STD_PAL_K, "PAL-K" }, { V4L2_STD_PAL_M, "PAL-M" }, { V4L2_STD_PAL_N, "PAL-N" }, { V4L2_STD_PAL_Nc, "PAL-Nc" }, { V4L2_STD_PAL_60, "PAL-60" }, { V4L2_STD_SECAM, "SECAM" }, { V4L2_STD_SECAM_B, "SECAM-B" }, { V4L2_STD_SECAM_G, "SECAM-G" }, { V4L2_STD_SECAM_H, "SECAM-H" }, { V4L2_STD_SECAM_DK, "SECAM-DK" }, { V4L2_STD_SECAM_D, "SECAM-D" }, { V4L2_STD_SECAM_K, "SECAM-K" }, { V4L2_STD_SECAM_K1, "SECAM-K1" }, { V4L2_STD_SECAM_L, "SECAM-L" }, { V4L2_STD_SECAM_LC, "SECAM-Lc" }, { 0, "Unknown" } }; /* video4linux standard ID conversion to standard name */ const char *v4l2_norm_to_name(v4l2_std_id id) { u32 myid = id; int i; /* HACK: ppc32 architecture doesn't have __ucmpdi2 function to handle 64 bit comparisons. So, on that architecture, with some gcc variants, compilation fails. Currently, the max value is 30bit wide. */ BUG_ON(myid != id); for (i = 0; standards[i].std; i++) if (myid == standards[i].std) break; return standards[i].descr; } EXPORT_SYMBOL(v4l2_norm_to_name); /* Returns frame period for the given standard */ void v4l2_video_std_frame_period(int id, struct v4l2_fract *frameperiod) { if (id & V4L2_STD_525_60) { frameperiod->numerator = 1001; frameperiod->denominator = 30000; } else { frameperiod->numerator = 1; frameperiod->denominator = 25; } } EXPORT_SYMBOL(v4l2_video_std_frame_period); /* Fill in the fields of a v4l2_standard structure according to the 'id' and 'transmission' parameters. Returns negative on error. */ int v4l2_video_std_construct(struct v4l2_standard *vs, int id, const char *name) { vs->id = id; v4l2_video_std_frame_period(id, &vs->frameperiod); vs->framelines = (id & V4L2_STD_525_60) ? 525 : 625; strscpy(vs->name, name, sizeof(vs->name)); return 0; } EXPORT_SYMBOL(v4l2_video_std_construct); /* Fill in the fields of a v4l2_standard structure according to the * 'id' and 'vs->index' parameters. Returns negative on error. */ int v4l_video_std_enumstd(struct v4l2_standard *vs, v4l2_std_id id) { v4l2_std_id curr_id = 0; unsigned int index = vs->index, i, j = 0; const char *descr = ""; /* Return -ENODATA if the id for the current input or output is 0, meaning that it doesn't support this API. */ if (id == 0) return -ENODATA; /* Return norm array in a canonical way */ for (i = 0; i <= index && id; i++) { /* last std value in the standards array is 0, so this while always ends there since (id & 0) == 0. */ while ((id & standards[j].std) != standards[j].std) j++; curr_id = standards[j].std; descr = standards[j].descr; j++; if (curr_id == 0) break; if (curr_id != V4L2_STD_PAL && curr_id != V4L2_STD_SECAM && curr_id != V4L2_STD_NTSC) id &= ~curr_id; } if (i <= index) return -EINVAL; v4l2_video_std_construct(vs, curr_id, descr); return 0; } /* ----------------------------------------------------------------- */ /* some arrays for pretty-printing debug messages of enum types */ const char *v4l2_field_names[] = { [V4L2_FIELD_ANY] = "any", [V4L2_FIELD_NONE] = "none", [V4L2_FIELD_TOP] = "top", [V4L2_FIELD_BOTTOM] = "bottom", [V4L2_FIELD_INTERLACED] = "interlaced", [V4L2_FIELD_SEQ_TB] = "seq-tb", [V4L2_FIELD_SEQ_BT] = "seq-bt", [V4L2_FIELD_ALTERNATE] = "alternate", [V4L2_FIELD_INTERLACED_TB] = "interlaced-tb", [V4L2_FIELD_INTERLACED_BT] = "interlaced-bt", }; EXPORT_SYMBOL(v4l2_field_names); const char *v4l2_type_names[] = { [0] = "0", [V4L2_BUF_TYPE_VIDEO_CAPTURE] = "vid-cap", [V4L2_BUF_TYPE_VIDEO_OVERLAY] = "vid-overlay", [V4L2_BUF_TYPE_VIDEO_OUTPUT] = "vid-out", [V4L2_BUF_TYPE_VBI_CAPTURE] = "vbi-cap", [V4L2_BUF_TYPE_VBI_OUTPUT] = "vbi-out", [V4L2_BUF_TYPE_SLICED_VBI_CAPTURE] = "sliced-vbi-cap", [V4L2_BUF_TYPE_SLICED_VBI_OUTPUT] = "sliced-vbi-out", [V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY] = "vid-out-overlay", [V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE] = "vid-cap-mplane", [V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE] = "vid-out-mplane", [V4L2_BUF_TYPE_SDR_CAPTURE] = "sdr-cap", [V4L2_BUF_TYPE_SDR_OUTPUT] = "sdr-out", [V4L2_BUF_TYPE_META_CAPTURE] = "meta-cap", [V4L2_BUF_TYPE_META_OUTPUT] = "meta-out", }; EXPORT_SYMBOL(v4l2_type_names); static const char *v4l2_memory_names[] = { [V4L2_MEMORY_MMAP] = "mmap", [V4L2_MEMORY_USERPTR] = "userptr", [V4L2_MEMORY_OVERLAY] = "overlay", [V4L2_MEMORY_DMABUF] = "dmabuf", }; #define prt_names(a, arr) (((unsigned)(a)) < ARRAY_SIZE(arr) ? arr[a] : "unknown") /* ------------------------------------------------------------------ */ /* debug help functions */ static void v4l_print_querycap(const void *arg, bool write_only) { const struct v4l2_capability *p = arg; pr_cont("driver=%.*s, card=%.*s, bus=%.*s, version=0x%08x, capabilities=0x%08x, device_caps=0x%08x\n", (int)sizeof(p->driver), p->driver, (int)sizeof(p->card), p->card, (int)sizeof(p->bus_info), p->bus_info, p->version, p->capabilities, p->device_caps); } static void v4l_print_enuminput(const void *arg, bool write_only) { const struct v4l2_input *p = arg; pr_cont("index=%u, name=%.*s, type=%u, audioset=0x%x, tuner=%u, std=0x%08Lx, status=0x%x, capabilities=0x%x\n", p->index, (int)sizeof(p->name), p->name, p->type, p->audioset, p->tuner, (unsigned long long)p->std, p->status, p->capabilities); } static void v4l_print_enumoutput(const void *arg, bool write_only) { const struct v4l2_output *p = arg; pr_cont("index=%u, name=%.*s, type=%u, audioset=0x%x, modulator=%u, std=0x%08Lx, capabilities=0x%x\n", p->index, (int)sizeof(p->name), p->name, p->type, p->audioset, p->modulator, (unsigned long long)p->std, p->capabilities); } static void v4l_print_audio(const void *arg, bool write_only) { const struct v4l2_audio *p = arg; if (write_only) pr_cont("index=%u, mode=0x%x\n", p->index, p->mode); else pr_cont("index=%u, name=%.*s, capability=0x%x, mode=0x%x\n", p->index, (int)sizeof(p->name), p->name, p->capability, p->mode); } static void v4l_print_audioout(const void *arg, bool write_only) { const struct v4l2_audioout *p = arg; if (write_only) pr_cont("index=%u\n", p->index); else pr_cont("index=%u, name=%.*s, capability=0x%x, mode=0x%x\n", p->index, (int)sizeof(p->name), p->name, p->capability, p->mode); } static void v4l_print_fmtdesc(const void *arg, bool write_only) { const struct v4l2_fmtdesc *p = arg; pr_cont("index=%u, type=%s, flags=0x%x, pixelformat=%p4cc, mbus_code=0x%04x, description='%.*s'\n", p->index, prt_names(p->type, v4l2_type_names), p->flags, &p->pixelformat, p->mbus_code, (int)sizeof(p->description), p->description); } static void v4l_print_format(const void *arg, bool write_only) { const struct v4l2_format *p = arg; const struct v4l2_pix_format *pix; const struct v4l2_pix_format_mplane *mp; const struct v4l2_vbi_format *vbi; const struct v4l2_sliced_vbi_format *sliced; const struct v4l2_window *win; const struct v4l2_meta_format *meta; u32 pixelformat; u32 planes; unsigned i; pr_cont("type=%s", prt_names(p->type, v4l2_type_names)); switch (p->type) { case V4L2_BUF_TYPE_VIDEO_CAPTURE: case V4L2_BUF_TYPE_VIDEO_OUTPUT: pix = &p->fmt.pix; pr_cont(", width=%u, height=%u, pixelformat=%p4cc, field=%s, bytesperline=%u, sizeimage=%u, colorspace=%d, flags=0x%x, ycbcr_enc=%u, quantization=%u, xfer_func=%u\n", pix->width, pix->height, &pix->pixelformat, prt_names(pix->field, v4l2_field_names), pix->bytesperline, pix->sizeimage, pix->colorspace, pix->flags, pix->ycbcr_enc, pix->quantization, pix->xfer_func); break; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: mp = &p->fmt.pix_mp; pixelformat = mp->pixelformat; pr_cont(", width=%u, height=%u, format=%p4cc, field=%s, colorspace=%d, num_planes=%u, flags=0x%x, ycbcr_enc=%u, quantization=%u, xfer_func=%u\n", mp->width, mp->height, &pixelformat, prt_names(mp->field, v4l2_field_names), mp->colorspace, mp->num_planes, mp->flags, mp->ycbcr_enc, mp->quantization, mp->xfer_func); planes = min_t(u32, mp->num_planes, VIDEO_MAX_PLANES); for (i = 0; i < planes; i++) printk(KERN_DEBUG "plane %u: bytesperline=%u sizeimage=%u\n", i, mp->plane_fmt[i].bytesperline, mp->plane_fmt[i].sizeimage); break; case V4L2_BUF_TYPE_VIDEO_OVERLAY: case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY: win = &p->fmt.win; pr_cont(", wxh=%dx%d, x,y=%d,%d, field=%s, chromakey=0x%08x, global_alpha=0x%02x\n", win->w.width, win->w.height, win->w.left, win->w.top, prt_names(win->field, v4l2_field_names), win->chromakey, win->global_alpha); break; case V4L2_BUF_TYPE_VBI_CAPTURE: case V4L2_BUF_TYPE_VBI_OUTPUT: vbi = &p->fmt.vbi; pr_cont(", sampling_rate=%u, offset=%u, samples_per_line=%u, sample_format=%p4cc, start=%u,%u, count=%u,%u\n", vbi->sampling_rate, vbi->offset, vbi->samples_per_line, &vbi->sample_format, vbi->start[0], vbi->start[1], vbi->count[0], vbi->count[1]); break; case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE: case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT: sliced = &p->fmt.sliced; pr_cont(", service_set=0x%08x, io_size=%d\n", sliced->service_set, sliced->io_size); for (i = 0; i < 24; i++) printk(KERN_DEBUG "line[%02u]=0x%04x, 0x%04x\n", i, sliced->service_lines[0][i], sliced->service_lines[1][i]); break; case V4L2_BUF_TYPE_SDR_CAPTURE: case V4L2_BUF_TYPE_SDR_OUTPUT: pixelformat = p->fmt.sdr.pixelformat; pr_cont(", pixelformat=%p4cc\n", &pixelformat); break; case V4L2_BUF_TYPE_META_CAPTURE: case V4L2_BUF_TYPE_META_OUTPUT: meta = &p->fmt.meta; pixelformat = meta->dataformat; pr_cont(", dataformat=%p4cc, buffersize=%u, width=%u, height=%u, bytesperline=%u\n", &pixelformat, meta->buffersize, meta->width, meta->height, meta->bytesperline); break; } } static void v4l_print_framebuffer(const void *arg, bool write_only) { const struct v4l2_framebuffer *p = arg; pr_cont("capability=0x%x, flags=0x%x, base=0x%p, width=%u, height=%u, pixelformat=%p4cc, bytesperline=%u, sizeimage=%u, colorspace=%d\n", p->capability, p->flags, p->base, p->fmt.width, p->fmt.height, &p->fmt.pixelformat, p->fmt.bytesperline, p->fmt.sizeimage, p->fmt.colorspace); } static void v4l_print_buftype(const void *arg, bool write_only) { pr_cont("type=%s\n", prt_names(*(u32 *)arg, v4l2_type_names)); } static void v4l_print_modulator(const void *arg, bool write_only) { const struct v4l2_modulator *p = arg; if (write_only) pr_cont("index=%u, txsubchans=0x%x\n", p->index, p->txsubchans); else pr_cont("index=%u, name=%.*s, capability=0x%x, rangelow=%u, rangehigh=%u, txsubchans=0x%x\n", p->index, (int)sizeof(p->name), p->name, p->capability, p->rangelow, p->rangehigh, p->txsubchans); } static void v4l_print_tuner(const void *arg, bool write_only) { const struct v4l2_tuner *p = arg; if (write_only) pr_cont("index=%u, audmode=%u\n", p->index, p->audmode); else pr_cont("index=%u, name=%.*s, type=%u, capability=0x%x, rangelow=%u, rangehigh=%u, signal=%u, afc=%d, rxsubchans=0x%x, audmode=%u\n", p->index, (int)sizeof(p->name), p->name, p->type, p->capability, p->rangelow, p->rangehigh, p->signal, p->afc, p->rxsubchans, p->audmode); } static void v4l_print_frequency(const void *arg, bool write_only) { const struct v4l2_frequency *p = arg; pr_cont("tuner=%u, type=%u, frequency=%u\n", p->tuner, p->type, p->frequency); } static void v4l_print_standard(const void *arg, bool write_only) { const struct v4l2_standard *p = arg; pr_cont("index=%u, id=0x%Lx, name=%.*s, fps=%u/%u, framelines=%u\n", p->index, (unsigned long long)p->id, (int)sizeof(p->name), p->name, p->frameperiod.numerator, p->frameperiod.denominator, p->framelines); } static void v4l_print_std(const void *arg, bool write_only) { pr_cont("std=0x%08Lx\n", *(const long long unsigned *)arg); } static void v4l_print_hw_freq_seek(const void *arg, bool write_only) { const struct v4l2_hw_freq_seek *p = arg; pr_cont("tuner=%u, type=%u, seek_upward=%u, wrap_around=%u, spacing=%u, rangelow=%u, rangehigh=%u\n", p->tuner, p->type, p->seek_upward, p->wrap_around, p->spacing, p->rangelow, p->rangehigh); } static void v4l_print_requestbuffers(const void *arg, bool write_only) { const struct v4l2_requestbuffers *p = arg; pr_cont("count=%d, type=%s, memory=%s\n", p->count, prt_names(p->type, v4l2_type_names), prt_names(p->memory, v4l2_memory_names)); } static void v4l_print_buffer(const void *arg, bool write_only) { const struct v4l2_buffer *p = arg; const struct v4l2_timecode *tc = &p->timecode; const struct v4l2_plane *plane; int i; pr_cont("%02d:%02d:%02d.%06ld index=%d, type=%s, request_fd=%d, flags=0x%08x, field=%s, sequence=%d, memory=%s", (int)p->timestamp.tv_sec / 3600, ((int)p->timestamp.tv_sec / 60) % 60, ((int)p->timestamp.tv_sec % 60), (long)p->timestamp.tv_usec, p->index, prt_names(p->type, v4l2_type_names), p->request_fd, p->flags, prt_names(p->field, v4l2_field_names), p->sequence, prt_names(p->memory, v4l2_memory_names)); if (V4L2_TYPE_IS_MULTIPLANAR(p->type) && p->m.planes) { pr_cont("\n"); for (i = 0; i < p->length; ++i) { plane = &p->m.planes[i]; printk(KERN_DEBUG "plane %d: bytesused=%d, data_offset=0x%08x, offset/userptr=0x%lx, length=%d\n", i, plane->bytesused, plane->data_offset, plane->m.userptr, plane->length); } } else { pr_cont(", bytesused=%d, offset/userptr=0x%lx, length=%d\n", p->bytesused, p->m.userptr, p->length); } printk(KERN_DEBUG "timecode=%02d:%02d:%02d type=%d, flags=0x%08x, frames=%d, userbits=0x%08x\n", tc->hours, tc->minutes, tc->seconds, tc->type, tc->flags, tc->frames, *(__u32 *)tc->userbits); } static void v4l_print_exportbuffer(const void *arg, bool write_only) { const struct v4l2_exportbuffer *p = arg; pr_cont("fd=%d, type=%s, index=%u, plane=%u, flags=0x%08x\n", p->fd, prt_names(p->type, v4l2_type_names), p->index, p->plane, p->flags); } static void v4l_print_create_buffers(const void *arg, bool write_only) { const struct v4l2_create_buffers *p = arg; pr_cont("index=%d, count=%d, memory=%s, capabilities=0x%08x, max num buffers=%u, ", p->index, p->count, prt_names(p->memory, v4l2_memory_names), p->capabilities, p->max_num_buffers); v4l_print_format(&p->format, write_only); } static void v4l_print_remove_buffers(const void *arg, bool write_only) { const struct v4l2_remove_buffers *p = arg; pr_cont("type=%s, index=%u, count=%u\n", prt_names(p->type, v4l2_type_names), p->index, p->count); } static void v4l_print_streamparm(const void *arg, bool write_only) { const struct v4l2_streamparm *p = arg; pr_cont("type=%s", prt_names(p->type, v4l2_type_names)); if (p->type == V4L2_BUF_TYPE_VIDEO_CAPTURE || p->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) { const struct v4l2_captureparm *c = &p->parm.capture; pr_cont(", capability=0x%x, capturemode=0x%x, timeperframe=%d/%d, extendedmode=%d, readbuffers=%d\n", c->capability, c->capturemode, c->timeperframe.numerator, c->timeperframe.denominator, c->extendedmode, c->readbuffers); } else if (p->type == V4L2_BUF_TYPE_VIDEO_OUTPUT || p->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) { const struct v4l2_outputparm *c = &p->parm.output; pr_cont(", capability=0x%x, outputmode=0x%x, timeperframe=%d/%d, extendedmode=%d, writebuffers=%d\n", c->capability, c->outputmode, c->timeperframe.numerator, c->timeperframe.denominator, c->extendedmode, c->writebuffers); } else { pr_cont("\n"); } } static void v4l_print_queryctrl(const void *arg, bool write_only) { const struct v4l2_queryctrl *p = arg; pr_cont("id=0x%x, type=%d, name=%.*s, min/max=%d/%d, step=%d, default=%d, flags=0x%08x\n", p->id, p->type, (int)sizeof(p->name), p->name, p->minimum, p->maximum, p->step, p->default_value, p->flags); } static void v4l_print_query_ext_ctrl(const void *arg, bool write_only) { const struct v4l2_query_ext_ctrl *p = arg; pr_cont("id=0x%x, type=%d, name=%.*s, min/max=%lld/%lld, step=%lld, default=%lld, flags=0x%08x, elem_size=%u, elems=%u, nr_of_dims=%u, dims=%u,%u,%u,%u\n", p->id, p->type, (int)sizeof(p->name), p->name, p->minimum, p->maximum, p->step, p->default_value, p->flags, p->elem_size, p->elems, p->nr_of_dims, p->dims[0], p->dims[1], p->dims[2], p->dims[3]); } static void v4l_print_querymenu(const void *arg, bool write_only) { const struct v4l2_querymenu *p = arg; pr_cont("id=0x%x, index=%d\n", p->id, p->index); } static void v4l_print_control(const void *arg, bool write_only) { const struct v4l2_control *p = arg; const char *name = v4l2_ctrl_get_name(p->id); if (name) pr_cont("name=%s, ", name); pr_cont("id=0x%x, value=%d\n", p->id, p->value); } static void v4l_print_ext_controls(const void *arg, bool write_only) { const struct v4l2_ext_controls *p = arg; int i; pr_cont("which=0x%x, count=%d, error_idx=%d, request_fd=%d", p->which, p->count, p->error_idx, p->request_fd); for (i = 0; i < p->count; i++) { unsigned int id = p->controls[i].id; const char *name = v4l2_ctrl_get_name(id); if (name) pr_cont(", name=%s", name); if (!p->controls[i].size) pr_cont(", id/val=0x%x/0x%x", id, p->controls[i].value); else pr_cont(", id/size=0x%x/%u", id, p->controls[i].size); } pr_cont("\n"); } static void v4l_print_cropcap(const void *arg, bool write_only) { const struct v4l2_cropcap *p = arg; pr_cont("type=%s, bounds wxh=%dx%d, x,y=%d,%d, defrect wxh=%dx%d, x,y=%d,%d, pixelaspect %d/%d\n", prt_names(p->type, v4l2_type_names), p->bounds.width, p->bounds.height, p->bounds.left, p->bounds.top, p->defrect.width, p->defrect.height, p->defrect.left, p->defrect.top, p->pixelaspect.numerator, p->pixelaspect.denominator); } static void v4l_print_crop(const void *arg, bool write_only) { const struct v4l2_crop *p = arg; pr_cont("type=%s, wxh=%dx%d, x,y=%d,%d\n", prt_names(p->type, v4l2_type_names), p->c.width, p->c.height, p->c.left, p->c.top); } static void v4l_print_selection(const void *arg, bool write_only) { const struct v4l2_selection *p = arg; pr_cont("type=%s, target=%d, flags=0x%x, wxh=%dx%d, x,y=%d,%d\n", prt_names(p->type, v4l2_type_names), p->target, p->flags, p->r.width, p->r.height, p->r.left, p->r.top); } static void v4l_print_jpegcompression(const void *arg, bool write_only) { const struct v4l2_jpegcompression *p = arg; pr_cont("quality=%d, APPn=%d, APP_len=%d, COM_len=%d, jpeg_markers=0x%x\n", p->quality, p->APPn, p->APP_len, p->COM_len, p->jpeg_markers); } static void v4l_print_enc_idx(const void *arg, bool write_only) { const struct v4l2_enc_idx *p = arg; pr_cont("entries=%d, entries_cap=%d\n", p->entries, p->entries_cap); } static void v4l_print_encoder_cmd(const void *arg, bool write_only) { const struct v4l2_encoder_cmd *p = arg; pr_cont("cmd=%d, flags=0x%x\n", p->cmd, p->flags); } static void v4l_print_decoder_cmd(const void *arg, bool write_only) { const struct v4l2_decoder_cmd *p = arg; pr_cont("cmd=%d, flags=0x%x\n", p->cmd, p->flags); if (p->cmd == V4L2_DEC_CMD_START) pr_info("speed=%d, format=%u\n", p->start.speed, p->start.format); else if (p->cmd == V4L2_DEC_CMD_STOP) pr_info("pts=%llu\n", p->stop.pts); } static void v4l_print_dbg_chip_info(const void *arg, bool write_only) { const struct v4l2_dbg_chip_info *p = arg; pr_cont("type=%u, ", p->match.type); if (p->match.type == V4L2_CHIP_MATCH_I2C_DRIVER) pr_cont("name=%.*s, ", (int)sizeof(p->match.name), p->match.name); else pr_cont("addr=%u, ", p->match.addr); pr_cont("name=%.*s\n", (int)sizeof(p->name), p->name); } static void v4l_print_dbg_register(const void *arg, bool write_only) { const struct v4l2_dbg_register *p = arg; pr_cont("type=%u, ", p->match.type); if (p->match.type == V4L2_CHIP_MATCH_I2C_DRIVER) pr_cont("name=%.*s, ", (int)sizeof(p->match.name), p->match.name); else pr_cont("addr=%u, ", p->match.addr); pr_cont("reg=0x%llx, val=0x%llx\n", p->reg, p->val); } static void v4l_print_dv_timings(const void *arg, bool write_only) { const struct v4l2_dv_timings *p = arg; switch (p->type) { case V4L2_DV_BT_656_1120: pr_cont("type=bt-656/1120, interlaced=%u, pixelclock=%llu, width=%u, height=%u, polarities=0x%x, hfrontporch=%u, hsync=%u, hbackporch=%u, vfrontporch=%u, vsync=%u, vbackporch=%u, il_vfrontporch=%u, il_vsync=%u, il_vbackporch=%u, standards=0x%x, flags=0x%x\n", p->bt.interlaced, p->bt.pixelclock, p->bt.width, p->bt.height, p->bt.polarities, p->bt.hfrontporch, p->bt.hsync, p->bt.hbackporch, p->bt.vfrontporch, p->bt.vsync, p->bt.vbackporch, p->bt.il_vfrontporch, p->bt.il_vsync, p->bt.il_vbackporch, p->bt.standards, p->bt.flags); break; default: pr_cont("type=%d\n", p->type); break; } } static void v4l_print_enum_dv_timings(const void *arg, bool write_only) { const struct v4l2_enum_dv_timings *p = arg; pr_cont("index=%u, ", p->index); v4l_print_dv_timings(&p->timings, write_only); } static void v4l_print_dv_timings_cap(const void *arg, bool write_only) { const struct v4l2_dv_timings_cap *p = arg; switch (p->type) { case V4L2_DV_BT_656_1120: pr_cont("type=bt-656/1120, width=%u-%u, height=%u-%u, pixelclock=%llu-%llu, standards=0x%x, capabilities=0x%x\n", p->bt.min_width, p->bt.max_width, p->bt.min_height, p->bt.max_height, p->bt.min_pixelclock, p->bt.max_pixelclock, p->bt.standards, p->bt.capabilities); break; default: pr_cont("type=%u\n", p->type); break; } } static void v4l_print_frmsizeenum(const void *arg, bool write_only) { const struct v4l2_frmsizeenum *p = arg; pr_cont("index=%u, pixelformat=%p4cc, type=%u", p->index, &p->pixel_format, p->type); switch (p->type) { case V4L2_FRMSIZE_TYPE_DISCRETE: pr_cont(", wxh=%ux%u\n", p->discrete.width, p->discrete.height); break; case V4L2_FRMSIZE_TYPE_STEPWISE: pr_cont(", min=%ux%u, max=%ux%u, step=%ux%u\n", p->stepwise.min_width, p->stepwise.min_height, p->stepwise.max_width, p->stepwise.max_height, p->stepwise.step_width, p->stepwise.step_height); break; case V4L2_FRMSIZE_TYPE_CONTINUOUS: default: pr_cont("\n"); break; } } static void v4l_print_frmivalenum(const void *arg, bool write_only) { const struct v4l2_frmivalenum *p = arg; pr_cont("index=%u, pixelformat=%p4cc, wxh=%ux%u, type=%u", p->index, &p->pixel_format, p->width, p->height, p->type); switch (p->type) { case V4L2_FRMIVAL_TYPE_DISCRETE: pr_cont(", fps=%d/%d\n", p->discrete.numerator, p->discrete.denominator); break; case V4L2_FRMIVAL_TYPE_STEPWISE: pr_cont(", min=%d/%d, max=%d/%d, step=%d/%d\n", p->stepwise.min.numerator, p->stepwise.min.denominator, p->stepwise.max.numerator, p->stepwise.max.denominator, p->stepwise.step.numerator, p->stepwise.step.denominator); break; case V4L2_FRMIVAL_TYPE_CONTINUOUS: default: pr_cont("\n"); break; } } static void v4l_print_event(const void *arg, bool write_only) { const struct v4l2_event *p = arg; const struct v4l2_event_ctrl *c; pr_cont("type=0x%x, pending=%u, sequence=%u, id=%u, timestamp=%llu.%9.9llu\n", p->type, p->pending, p->sequence, p->id, p->timestamp.tv_sec, p->timestamp.tv_nsec); switch (p->type) { case V4L2_EVENT_VSYNC: printk(KERN_DEBUG "field=%s\n", prt_names(p->u.vsync.field, v4l2_field_names)); break; case V4L2_EVENT_CTRL: c = &p->u.ctrl; printk(KERN_DEBUG "changes=0x%x, type=%u, ", c->changes, c->type); if (c->type == V4L2_CTRL_TYPE_INTEGER64) pr_cont("value64=%lld, ", c->value64); else pr_cont("value=%d, ", c->value); pr_cont("flags=0x%x, minimum=%d, maximum=%d, step=%d, default_value=%d\n", c->flags, c->minimum, c->maximum, c->step, c->default_value); break; case V4L2_EVENT_FRAME_SYNC: pr_cont("frame_sequence=%u\n", p->u.frame_sync.frame_sequence); break; } } static void v4l_print_event_subscription(const void *arg, bool write_only) { const struct v4l2_event_subscription *p = arg; pr_cont("type=0x%x, id=0x%x, flags=0x%x\n", p->type, p->id, p->flags); } static void v4l_print_sliced_vbi_cap(const void *arg, bool write_only) { const struct v4l2_sliced_vbi_cap *p = arg; int i; pr_cont("type=%s, service_set=0x%08x\n", prt_names(p->type, v4l2_type_names), p->service_set); for (i = 0; i < 24; i++) printk(KERN_DEBUG "line[%02u]=0x%04x, 0x%04x\n", i, p->service_lines[0][i], p->service_lines[1][i]); } static void v4l_print_freq_band(const void *arg, bool write_only) { const struct v4l2_frequency_band *p = arg; pr_cont("tuner=%u, type=%u, index=%u, capability=0x%x, rangelow=%u, rangehigh=%u, modulation=0x%x\n", p->tuner, p->type, p->index, p->capability, p->rangelow, p->rangehigh, p->modulation); } static void v4l_print_edid(const void *arg, bool write_only) { const struct v4l2_edid *p = arg; pr_cont("pad=%u, start_block=%u, blocks=%u\n", p->pad, p->start_block, p->blocks); } static void v4l_print_u32(const void *arg, bool write_only) { pr_cont("value=%u\n", *(const u32 *)arg); } static void v4l_print_newline(const void *arg, bool write_only) { pr_cont("\n"); } static void v4l_print_default(const void *arg, bool write_only) { pr_cont("driver-specific ioctl\n"); } static bool check_ext_ctrls(struct v4l2_ext_controls *c, unsigned long ioctl) { __u32 i; /* zero the reserved fields */ c->reserved[0] = 0; for (i = 0; i < c->count; i++) c->controls[i].reserved2[0] = 0; switch (c->which) { case V4L2_CID_PRIVATE_BASE: /* * V4L2_CID_PRIVATE_BASE cannot be used as control class * when using extended controls. * Only when passed in through VIDIOC_G_CTRL and VIDIOC_S_CTRL * is it allowed for backwards compatibility. */ if (ioctl == VIDIOC_G_CTRL || ioctl == VIDIOC_S_CTRL) return false; break; case V4L2_CTRL_WHICH_DEF_VAL: /* Default value cannot be changed */ if (ioctl == VIDIOC_S_EXT_CTRLS || ioctl == VIDIOC_TRY_EXT_CTRLS) { c->error_idx = c->count; return false; } return true; case V4L2_CTRL_WHICH_CUR_VAL: return true; case V4L2_CTRL_WHICH_REQUEST_VAL: c->error_idx = c->count; return false; } /* Check that all controls are from the same control class. */ for (i = 0; i < c->count; i++) { if (V4L2_CTRL_ID2WHICH(c->controls[i].id) != c->which) { c->error_idx = ioctl == VIDIOC_TRY_EXT_CTRLS ? i : c->count; return false; } } return true; } static int check_fmt(struct file *file, enum v4l2_buf_type type) { const u32 vid_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_VIDEO_OUTPUT | V4L2_CAP_VIDEO_OUTPUT_MPLANE | V4L2_CAP_VIDEO_M2M | V4L2_CAP_VIDEO_M2M_MPLANE; const u32 meta_caps = V4L2_CAP_META_CAPTURE | V4L2_CAP_META_OUTPUT; struct video_device *vfd = video_devdata(file); const struct v4l2_ioctl_ops *ops = vfd->ioctl_ops; bool is_vid = vfd->vfl_type == VFL_TYPE_VIDEO && (vfd->device_caps & vid_caps); bool is_vbi = vfd->vfl_type == VFL_TYPE_VBI; bool is_sdr = vfd->vfl_type == VFL_TYPE_SDR; bool is_tch = vfd->vfl_type == VFL_TYPE_TOUCH; bool is_meta = vfd->vfl_type == VFL_TYPE_VIDEO && (vfd->device_caps & meta_caps); bool is_rx = vfd->vfl_dir != VFL_DIR_TX; bool is_tx = vfd->vfl_dir != VFL_DIR_RX; if (ops == NULL) return -EINVAL; switch (type) { case V4L2_BUF_TYPE_VIDEO_CAPTURE: if ((is_vid || is_tch) && is_rx && (ops->vidioc_g_fmt_vid_cap || ops->vidioc_g_fmt_vid_cap_mplane)) return 0; break; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: if ((is_vid || is_tch) && is_rx && ops->vidioc_g_fmt_vid_cap_mplane) return 0; break; case V4L2_BUF_TYPE_VIDEO_OVERLAY: if (is_vid && is_rx && ops->vidioc_g_fmt_vid_overlay) return 0; break; case V4L2_BUF_TYPE_VIDEO_OUTPUT: if (is_vid && is_tx && (ops->vidioc_g_fmt_vid_out || ops->vidioc_g_fmt_vid_out_mplane)) return 0; break; case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: if (is_vid && is_tx && ops->vidioc_g_fmt_vid_out_mplane) return 0; break; case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY: if (is_vid && is_tx && ops->vidioc_g_fmt_vid_out_overlay) return 0; break; case V4L2_BUF_TYPE_VBI_CAPTURE: if (is_vbi && is_rx && ops->vidioc_g_fmt_vbi_cap) return 0; break; case V4L2_BUF_TYPE_VBI_OUTPUT: if (is_vbi && is_tx && ops->vidioc_g_fmt_vbi_out) return 0; break; case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE: if (is_vbi && is_rx && ops->vidioc_g_fmt_sliced_vbi_cap) return 0; break; case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT: if (is_vbi && is_tx && ops->vidioc_g_fmt_sliced_vbi_out) return 0; break; case V4L2_BUF_TYPE_SDR_CAPTURE: if (is_sdr && is_rx && ops->vidioc_g_fmt_sdr_cap) return 0; break; case V4L2_BUF_TYPE_SDR_OUTPUT: if (is_sdr && is_tx && ops->vidioc_g_fmt_sdr_out) return 0; break; case V4L2_BUF_TYPE_META_CAPTURE: if (is_meta && is_rx && ops->vidioc_g_fmt_meta_cap) return 0; break; case V4L2_BUF_TYPE_META_OUTPUT: if (is_meta && is_tx && ops->vidioc_g_fmt_meta_out) return 0; break; default: break; } return -EINVAL; } static void v4l_sanitize_colorspace(u32 pixelformat, u32 *colorspace, u32 *encoding, u32 *quantization, u32 *xfer_func) { bool is_hsv = pixelformat == V4L2_PIX_FMT_HSV24 || pixelformat == V4L2_PIX_FMT_HSV32; if (!v4l2_is_colorspace_valid(*colorspace)) { *colorspace = V4L2_COLORSPACE_DEFAULT; *encoding = V4L2_YCBCR_ENC_DEFAULT; *quantization = V4L2_QUANTIZATION_DEFAULT; *xfer_func = V4L2_XFER_FUNC_DEFAULT; } if ((!is_hsv && !v4l2_is_ycbcr_enc_valid(*encoding)) || (is_hsv && !v4l2_is_hsv_enc_valid(*encoding))) *encoding = V4L2_YCBCR_ENC_DEFAULT; if (!v4l2_is_quant_valid(*quantization)) *quantization = V4L2_QUANTIZATION_DEFAULT; if (!v4l2_is_xfer_func_valid(*xfer_func)) *xfer_func = V4L2_XFER_FUNC_DEFAULT; } static void v4l_sanitize_format(struct v4l2_format *fmt) { unsigned int offset; /* Make sure num_planes is not bogus */ if (fmt->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE || fmt->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) fmt->fmt.pix_mp.num_planes = min_t(u32, fmt->fmt.pix_mp.num_planes, VIDEO_MAX_PLANES); /* * The v4l2_pix_format structure has been extended with fields that were * not previously required to be set to zero by applications. The priv * field, when set to a magic value, indicates that the extended fields * are valid. Otherwise they will contain undefined values. To simplify * the API towards drivers zero the extended fields and set the priv * field to the magic value when the extended pixel format structure * isn't used by applications. */ if (fmt->type == V4L2_BUF_TYPE_VIDEO_CAPTURE || fmt->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) { if (fmt->fmt.pix.priv != V4L2_PIX_FMT_PRIV_MAGIC) { fmt->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC; offset = offsetof(struct v4l2_pix_format, priv) + sizeof(fmt->fmt.pix.priv); memset(((void *)&fmt->fmt.pix) + offset, 0, sizeof(fmt->fmt.pix) - offset); } } /* Replace invalid colorspace values with defaults. */ if (fmt->type == V4L2_BUF_TYPE_VIDEO_CAPTURE || fmt->type == V4L2_BUF_TYPE_VIDEO_OUTPUT) { v4l_sanitize_colorspace(fmt->fmt.pix.pixelformat, &fmt->fmt.pix.colorspace, &fmt->fmt.pix.ycbcr_enc, &fmt->fmt.pix.quantization, &fmt->fmt.pix.xfer_func); } else if (fmt->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE || fmt->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) { u32 ycbcr_enc = fmt->fmt.pix_mp.ycbcr_enc; u32 quantization = fmt->fmt.pix_mp.quantization; u32 xfer_func = fmt->fmt.pix_mp.xfer_func; v4l_sanitize_colorspace(fmt->fmt.pix_mp.pixelformat, &fmt->fmt.pix_mp.colorspace, &ycbcr_enc, &quantization, &xfer_func); fmt->fmt.pix_mp.ycbcr_enc = ycbcr_enc; fmt->fmt.pix_mp.quantization = quantization; fmt->fmt.pix_mp.xfer_func = xfer_func; } } static int v4l_querycap(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_capability *cap = (struct v4l2_capability *)arg; struct video_device *vfd = video_devdata(file); int ret; cap->version = LINUX_VERSION_CODE; cap->device_caps = vfd->device_caps; cap->capabilities = vfd->device_caps | V4L2_CAP_DEVICE_CAPS; media_set_bus_info(cap->bus_info, sizeof(cap->bus_info), vfd->dev_parent); ret = ops->vidioc_querycap(file, fh, cap); /* * Drivers must not change device_caps, so check for this and * warn if this happened. */ WARN_ON(cap->device_caps != vfd->device_caps); /* * Check that capabilities is a superset of * vfd->device_caps | V4L2_CAP_DEVICE_CAPS */ WARN_ON((cap->capabilities & (vfd->device_caps | V4L2_CAP_DEVICE_CAPS)) != (vfd->device_caps | V4L2_CAP_DEVICE_CAPS)); cap->capabilities |= V4L2_CAP_EXT_PIX_FORMAT; cap->device_caps |= V4L2_CAP_EXT_PIX_FORMAT; return ret; } static int v4l_g_input(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); if (vfd->device_caps & V4L2_CAP_IO_MC) { *(int *)arg = 0; return 0; } return ops->vidioc_g_input(file, fh, arg); } static int v4l_g_output(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); if (vfd->device_caps & V4L2_CAP_IO_MC) { *(int *)arg = 0; return 0; } return ops->vidioc_g_output(file, fh, arg); } static int v4l_s_input(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); int ret; ret = v4l_enable_media_source(vfd); if (ret) return ret; if (vfd->device_caps & V4L2_CAP_IO_MC) return *(int *)arg ? -EINVAL : 0; return ops->vidioc_s_input(file, fh, *(unsigned int *)arg); } static int v4l_s_output(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); if (vfd->device_caps & V4L2_CAP_IO_MC) return *(int *)arg ? -EINVAL : 0; return ops->vidioc_s_output(file, fh, *(unsigned int *)arg); } static int v4l_g_priority(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd; u32 *p = arg; vfd = video_devdata(file); *p = v4l2_prio_max(vfd->prio); return 0; } static int v4l_s_priority(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd; struct v4l2_fh *vfh; u32 *p = arg; vfd = video_devdata(file); if (!test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags)) return -ENOTTY; vfh = file->private_data; return v4l2_prio_change(vfd->prio, &vfh->prio, *p); } static int v4l_enuminput(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_input *p = arg; /* * We set the flags for CAP_DV_TIMINGS & * CAP_STD here based on ioctl handler provided by the * driver. If the driver doesn't support these * for a specific input, it must override these flags. */ if (is_valid_ioctl(vfd, VIDIOC_S_STD)) p->capabilities |= V4L2_IN_CAP_STD; if (vfd->device_caps & V4L2_CAP_IO_MC) { if (p->index) return -EINVAL; strscpy(p->name, vfd->name, sizeof(p->name)); p->type = V4L2_INPUT_TYPE_CAMERA; return 0; } return ops->vidioc_enum_input(file, fh, p); } static int v4l_enumoutput(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_output *p = arg; /* * We set the flags for CAP_DV_TIMINGS & * CAP_STD here based on ioctl handler provided by the * driver. If the driver doesn't support these * for a specific output, it must override these flags. */ if (is_valid_ioctl(vfd, VIDIOC_S_STD)) p->capabilities |= V4L2_OUT_CAP_STD; if (vfd->device_caps & V4L2_CAP_IO_MC) { if (p->index) return -EINVAL; strscpy(p->name, vfd->name, sizeof(p->name)); p->type = V4L2_OUTPUT_TYPE_ANALOG; return 0; } return ops->vidioc_enum_output(file, fh, p); } static void v4l_fill_fmtdesc(struct v4l2_fmtdesc *fmt) { const unsigned sz = sizeof(fmt->description); const char *descr = NULL; u32 flags = 0; /* * We depart from the normal coding style here since the descriptions * should be aligned so it is easy to see which descriptions will be * longer than 31 characters (the max length for a description). * And frankly, this is easier to read anyway. * * Note that gcc will use O(log N) comparisons to find the right case. */ switch (fmt->pixelformat) { /* Max description length mask: descr = "0123456789012345678901234567890" */ case V4L2_PIX_FMT_RGB332: descr = "8-bit RGB 3-3-2"; break; case V4L2_PIX_FMT_RGB444: descr = "16-bit A/XRGB 4-4-4-4"; break; case V4L2_PIX_FMT_ARGB444: descr = "16-bit ARGB 4-4-4-4"; break; case V4L2_PIX_FMT_XRGB444: descr = "16-bit XRGB 4-4-4-4"; break; case V4L2_PIX_FMT_RGBA444: descr = "16-bit RGBA 4-4-4-4"; break; case V4L2_PIX_FMT_RGBX444: descr = "16-bit RGBX 4-4-4-4"; break; case V4L2_PIX_FMT_ABGR444: descr = "16-bit ABGR 4-4-4-4"; break; case V4L2_PIX_FMT_XBGR444: descr = "16-bit XBGR 4-4-4-4"; break; case V4L2_PIX_FMT_BGRA444: descr = "16-bit BGRA 4-4-4-4"; break; case V4L2_PIX_FMT_BGRX444: descr = "16-bit BGRX 4-4-4-4"; break; case V4L2_PIX_FMT_RGB555: descr = "16-bit A/XRGB 1-5-5-5"; break; case V4L2_PIX_FMT_ARGB555: descr = "16-bit ARGB 1-5-5-5"; break; case V4L2_PIX_FMT_XRGB555: descr = "16-bit XRGB 1-5-5-5"; break; case V4L2_PIX_FMT_ABGR555: descr = "16-bit ABGR 1-5-5-5"; break; case V4L2_PIX_FMT_XBGR555: descr = "16-bit XBGR 1-5-5-5"; break; case V4L2_PIX_FMT_RGBA555: descr = "16-bit RGBA 5-5-5-1"; break; case V4L2_PIX_FMT_RGBX555: descr = "16-bit RGBX 5-5-5-1"; break; case V4L2_PIX_FMT_BGRA555: descr = "16-bit BGRA 5-5-5-1"; break; case V4L2_PIX_FMT_BGRX555: descr = "16-bit BGRX 5-5-5-1"; break; case V4L2_PIX_FMT_RGB565: descr = "16-bit RGB 5-6-5"; break; case V4L2_PIX_FMT_RGB555X: descr = "16-bit A/XRGB 1-5-5-5 BE"; break; case V4L2_PIX_FMT_ARGB555X: descr = "16-bit ARGB 1-5-5-5 BE"; break; case V4L2_PIX_FMT_XRGB555X: descr = "16-bit XRGB 1-5-5-5 BE"; break; case V4L2_PIX_FMT_RGB565X: descr = "16-bit RGB 5-6-5 BE"; break; case V4L2_PIX_FMT_BGR666: descr = "18-bit BGRX 6-6-6-14"; break; case V4L2_PIX_FMT_BGR24: descr = "24-bit BGR 8-8-8"; break; case V4L2_PIX_FMT_RGB24: descr = "24-bit RGB 8-8-8"; break; case V4L2_PIX_FMT_BGR32: descr = "32-bit BGRA/X 8-8-8-8"; break; case V4L2_PIX_FMT_ABGR32: descr = "32-bit BGRA 8-8-8-8"; break; case V4L2_PIX_FMT_XBGR32: descr = "32-bit BGRX 8-8-8-8"; break; case V4L2_PIX_FMT_RGB32: descr = "32-bit A/XRGB 8-8-8-8"; break; case V4L2_PIX_FMT_ARGB32: descr = "32-bit ARGB 8-8-8-8"; break; case V4L2_PIX_FMT_XRGB32: descr = "32-bit XRGB 8-8-8-8"; break; case V4L2_PIX_FMT_BGRA32: descr = "32-bit ABGR 8-8-8-8"; break; case V4L2_PIX_FMT_BGRX32: descr = "32-bit XBGR 8-8-8-8"; break; case V4L2_PIX_FMT_RGBA32: descr = "32-bit RGBA 8-8-8-8"; break; case V4L2_PIX_FMT_RGBX32: descr = "32-bit RGBX 8-8-8-8"; break; case V4L2_PIX_FMT_RGBX1010102: descr = "32-bit RGBX 10-10-10-2"; break; case V4L2_PIX_FMT_RGBA1010102: descr = "32-bit RGBA 10-10-10-2"; break; case V4L2_PIX_FMT_ARGB2101010: descr = "32-bit ARGB 2-10-10-10"; break; case V4L2_PIX_FMT_BGR48: descr = "48-bit BGR 16-16-16"; break; case V4L2_PIX_FMT_RGB48: descr = "48-bit RGB 16-16-16"; break; case V4L2_PIX_FMT_BGR48_12: descr = "12-bit Depth BGR"; break; case V4L2_PIX_FMT_ABGR64_12: descr = "12-bit Depth BGRA"; break; case V4L2_PIX_FMT_GREY: descr = "8-bit Greyscale"; break; case V4L2_PIX_FMT_Y4: descr = "4-bit Greyscale"; break; case V4L2_PIX_FMT_Y6: descr = "6-bit Greyscale"; break; case V4L2_PIX_FMT_Y10: descr = "10-bit Greyscale"; break; case V4L2_PIX_FMT_Y12: descr = "12-bit Greyscale"; break; case V4L2_PIX_FMT_Y012: descr = "12-bit Greyscale (bits 15-4)"; break; case V4L2_PIX_FMT_Y14: descr = "14-bit Greyscale"; break; case V4L2_PIX_FMT_Y16: descr = "16-bit Greyscale"; break; case V4L2_PIX_FMT_Y16_BE: descr = "16-bit Greyscale BE"; break; case V4L2_PIX_FMT_Y10BPACK: descr = "10-bit Greyscale (Packed)"; break; case V4L2_PIX_FMT_Y10P: descr = "10-bit Greyscale (MIPI Packed)"; break; case V4L2_PIX_FMT_IPU3_Y10: descr = "10-bit greyscale (IPU3 Packed)"; break; case V4L2_PIX_FMT_Y12P: descr = "12-bit Greyscale (MIPI Packed)"; break; case V4L2_PIX_FMT_Y14P: descr = "14-bit Greyscale (MIPI Packed)"; break; case V4L2_PIX_FMT_Y8I: descr = "Interleaved 8-bit Greyscale"; break; case V4L2_PIX_FMT_Y12I: descr = "Interleaved 12-bit Greyscale"; break; case V4L2_PIX_FMT_Y16I: descr = "Interleaved 16-bit Greyscale"; break; case V4L2_PIX_FMT_Z16: descr = "16-bit Depth"; break; case V4L2_PIX_FMT_INZI: descr = "Planar 10:16 Greyscale Depth"; break; case V4L2_PIX_FMT_CNF4: descr = "4-bit Depth Confidence (Packed)"; break; case V4L2_PIX_FMT_PAL8: descr = "8-bit Palette"; break; case V4L2_PIX_FMT_UV8: descr = "8-bit Chrominance UV 4-4"; break; case V4L2_PIX_FMT_YVU410: descr = "Planar YVU 4:1:0"; break; case V4L2_PIX_FMT_YVU420: descr = "Planar YVU 4:2:0"; break; case V4L2_PIX_FMT_YUYV: descr = "YUYV 4:2:2"; break; case V4L2_PIX_FMT_YYUV: descr = "YYUV 4:2:2"; break; case V4L2_PIX_FMT_YVYU: descr = "YVYU 4:2:2"; break; case V4L2_PIX_FMT_UYVY: descr = "UYVY 4:2:2"; break; case V4L2_PIX_FMT_VYUY: descr = "VYUY 4:2:2"; break; case V4L2_PIX_FMT_YUV422P: descr = "Planar YUV 4:2:2"; break; case V4L2_PIX_FMT_YUV411P: descr = "Planar YUV 4:1:1"; break; case V4L2_PIX_FMT_Y41P: descr = "YUV 4:1:1 (Packed)"; break; case V4L2_PIX_FMT_YUV444: descr = "16-bit A/XYUV 4-4-4-4"; break; case V4L2_PIX_FMT_YUV555: descr = "16-bit A/XYUV 1-5-5-5"; break; case V4L2_PIX_FMT_YUV565: descr = "16-bit YUV 5-6-5"; break; case V4L2_PIX_FMT_YUV24: descr = "24-bit YUV 4:4:4 8-8-8"; break; case V4L2_PIX_FMT_YUV32: descr = "32-bit A/XYUV 8-8-8-8"; break; case V4L2_PIX_FMT_AYUV32: descr = "32-bit AYUV 8-8-8-8"; break; case V4L2_PIX_FMT_XYUV32: descr = "32-bit XYUV 8-8-8-8"; break; case V4L2_PIX_FMT_VUYA32: descr = "32-bit VUYA 8-8-8-8"; break; case V4L2_PIX_FMT_VUYX32: descr = "32-bit VUYX 8-8-8-8"; break; case V4L2_PIX_FMT_YUVA32: descr = "32-bit YUVA 8-8-8-8"; break; case V4L2_PIX_FMT_YUVX32: descr = "32-bit YUVX 8-8-8-8"; break; case V4L2_PIX_FMT_YUV410: descr = "Planar YUV 4:1:0"; break; case V4L2_PIX_FMT_YUV420: descr = "Planar YUV 4:2:0"; break; case V4L2_PIX_FMT_HI240: descr = "8-bit Dithered RGB (BTTV)"; break; case V4L2_PIX_FMT_M420: descr = "YUV 4:2:0 (M420)"; break; case V4L2_PIX_FMT_YUV48_12: descr = "12-bit YUV 4:4:4 Packed"; break; case V4L2_PIX_FMT_NV12: descr = "Y/UV 4:2:0"; break; case V4L2_PIX_FMT_NV21: descr = "Y/VU 4:2:0"; break; case V4L2_PIX_FMT_NV16: descr = "Y/UV 4:2:2"; break; case V4L2_PIX_FMT_NV61: descr = "Y/VU 4:2:2"; break; case V4L2_PIX_FMT_NV24: descr = "Y/UV 4:4:4"; break; case V4L2_PIX_FMT_NV42: descr = "Y/VU 4:4:4"; break; case V4L2_PIX_FMT_P010: descr = "10-bit Y/UV 4:2:0"; break; case V4L2_PIX_FMT_P012: descr = "12-bit Y/UV 4:2:0"; break; case V4L2_PIX_FMT_NV12_4L4: descr = "Y/UV 4:2:0 (4x4 Linear)"; break; case V4L2_PIX_FMT_NV12_16L16: descr = "Y/UV 4:2:0 (16x16 Linear)"; break; case V4L2_PIX_FMT_NV12_32L32: descr = "Y/UV 4:2:0 (32x32 Linear)"; break; case V4L2_PIX_FMT_NV15_4L4: descr = "10-bit Y/UV 4:2:0 (4x4 Linear)"; break; case V4L2_PIX_FMT_P010_4L4: descr = "10-bit Y/UV 4:2:0 (4x4 Linear)"; break; case V4L2_PIX_FMT_NV12M: descr = "Y/UV 4:2:0 (N-C)"; break; case V4L2_PIX_FMT_NV21M: descr = "Y/VU 4:2:0 (N-C)"; break; case V4L2_PIX_FMT_NV16M: descr = "Y/UV 4:2:2 (N-C)"; break; case V4L2_PIX_FMT_NV61M: descr = "Y/VU 4:2:2 (N-C)"; break; case V4L2_PIX_FMT_NV12MT: descr = "Y/UV 4:2:0 (64x32 MB, N-C)"; break; case V4L2_PIX_FMT_NV12MT_16X16: descr = "Y/UV 4:2:0 (16x16 MB, N-C)"; break; case V4L2_PIX_FMT_P012M: descr = "12-bit Y/UV 4:2:0 (N-C)"; break; case V4L2_PIX_FMT_YUV420M: descr = "Planar YUV 4:2:0 (N-C)"; break; case V4L2_PIX_FMT_YVU420M: descr = "Planar YVU 4:2:0 (N-C)"; break; case V4L2_PIX_FMT_YUV422M: descr = "Planar YUV 4:2:2 (N-C)"; break; case V4L2_PIX_FMT_YVU422M: descr = "Planar YVU 4:2:2 (N-C)"; break; case V4L2_PIX_FMT_YUV444M: descr = "Planar YUV 4:4:4 (N-C)"; break; case V4L2_PIX_FMT_YVU444M: descr = "Planar YVU 4:4:4 (N-C)"; break; case V4L2_PIX_FMT_SBGGR8: descr = "8-bit Bayer BGBG/GRGR"; break; case V4L2_PIX_FMT_SGBRG8: descr = "8-bit Bayer GBGB/RGRG"; break; case V4L2_PIX_FMT_SGRBG8: descr = "8-bit Bayer GRGR/BGBG"; break; case V4L2_PIX_FMT_SRGGB8: descr = "8-bit Bayer RGRG/GBGB"; break; case V4L2_PIX_FMT_SBGGR10: descr = "10-bit Bayer BGBG/GRGR"; break; case V4L2_PIX_FMT_SGBRG10: descr = "10-bit Bayer GBGB/RGRG"; break; case V4L2_PIX_FMT_SGRBG10: descr = "10-bit Bayer GRGR/BGBG"; break; case V4L2_PIX_FMT_SRGGB10: descr = "10-bit Bayer RGRG/GBGB"; break; case V4L2_PIX_FMT_SBGGR10P: descr = "10-bit Bayer BGBG/GRGR Packed"; break; case V4L2_PIX_FMT_SGBRG10P: descr = "10-bit Bayer GBGB/RGRG Packed"; break; case V4L2_PIX_FMT_SGRBG10P: descr = "10-bit Bayer GRGR/BGBG Packed"; break; case V4L2_PIX_FMT_SRGGB10P: descr = "10-bit Bayer RGRG/GBGB Packed"; break; case V4L2_PIX_FMT_IPU3_SBGGR10: descr = "10-bit bayer BGGR IPU3 Packed"; break; case V4L2_PIX_FMT_IPU3_SGBRG10: descr = "10-bit bayer GBRG IPU3 Packed"; break; case V4L2_PIX_FMT_IPU3_SGRBG10: descr = "10-bit bayer GRBG IPU3 Packed"; break; case V4L2_PIX_FMT_IPU3_SRGGB10: descr = "10-bit bayer RGGB IPU3 Packed"; break; case V4L2_PIX_FMT_SBGGR10ALAW8: descr = "8-bit Bayer BGBG/GRGR (A-law)"; break; case V4L2_PIX_FMT_SGBRG10ALAW8: descr = "8-bit Bayer GBGB/RGRG (A-law)"; break; case V4L2_PIX_FMT_SGRBG10ALAW8: descr = "8-bit Bayer GRGR/BGBG (A-law)"; break; case V4L2_PIX_FMT_SRGGB10ALAW8: descr = "8-bit Bayer RGRG/GBGB (A-law)"; break; case V4L2_PIX_FMT_SBGGR10DPCM8: descr = "8-bit Bayer BGBG/GRGR (DPCM)"; break; case V4L2_PIX_FMT_SGBRG10DPCM8: descr = "8-bit Bayer GBGB/RGRG (DPCM)"; break; case V4L2_PIX_FMT_SGRBG10DPCM8: descr = "8-bit Bayer GRGR/BGBG (DPCM)"; break; case V4L2_PIX_FMT_SRGGB10DPCM8: descr = "8-bit Bayer RGRG/GBGB (DPCM)"; break; case V4L2_PIX_FMT_SBGGR12: descr = "12-bit Bayer BGBG/GRGR"; break; case V4L2_PIX_FMT_SGBRG12: descr = "12-bit Bayer GBGB/RGRG"; break; case V4L2_PIX_FMT_SGRBG12: descr = "12-bit Bayer GRGR/BGBG"; break; case V4L2_PIX_FMT_SRGGB12: descr = "12-bit Bayer RGRG/GBGB"; break; case V4L2_PIX_FMT_SBGGR12P: descr = "12-bit Bayer BGBG/GRGR Packed"; break; case V4L2_PIX_FMT_SGBRG12P: descr = "12-bit Bayer GBGB/RGRG Packed"; break; case V4L2_PIX_FMT_SGRBG12P: descr = "12-bit Bayer GRGR/BGBG Packed"; break; case V4L2_PIX_FMT_SRGGB12P: descr = "12-bit Bayer RGRG/GBGB Packed"; break; case V4L2_PIX_FMT_SBGGR14: descr = "14-bit Bayer BGBG/GRGR"; break; case V4L2_PIX_FMT_SGBRG14: descr = "14-bit Bayer GBGB/RGRG"; break; case V4L2_PIX_FMT_SGRBG14: descr = "14-bit Bayer GRGR/BGBG"; break; case V4L2_PIX_FMT_SRGGB14: descr = "14-bit Bayer RGRG/GBGB"; break; case V4L2_PIX_FMT_SBGGR14P: descr = "14-bit Bayer BGBG/GRGR Packed"; break; case V4L2_PIX_FMT_SGBRG14P: descr = "14-bit Bayer GBGB/RGRG Packed"; break; case V4L2_PIX_FMT_SGRBG14P: descr = "14-bit Bayer GRGR/BGBG Packed"; break; case V4L2_PIX_FMT_SRGGB14P: descr = "14-bit Bayer RGRG/GBGB Packed"; break; case V4L2_PIX_FMT_SBGGR16: descr = "16-bit Bayer BGBG/GRGR"; break; case V4L2_PIX_FMT_SGBRG16: descr = "16-bit Bayer GBGB/RGRG"; break; case V4L2_PIX_FMT_SGRBG16: descr = "16-bit Bayer GRGR/BGBG"; break; case V4L2_PIX_FMT_SRGGB16: descr = "16-bit Bayer RGRG/GBGB"; break; case V4L2_PIX_FMT_SN9C20X_I420: descr = "GSPCA SN9C20X I420"; break; case V4L2_PIX_FMT_SPCA501: descr = "GSPCA SPCA501"; break; case V4L2_PIX_FMT_SPCA505: descr = "GSPCA SPCA505"; break; case V4L2_PIX_FMT_SPCA508: descr = "GSPCA SPCA508"; break; case V4L2_PIX_FMT_STV0680: descr = "GSPCA STV0680"; break; case V4L2_PIX_FMT_TM6000: descr = "A/V + VBI Mux Packet"; break; case V4L2_PIX_FMT_CIT_YYVYUY: descr = "GSPCA CIT YYVYUY"; break; case V4L2_PIX_FMT_KONICA420: descr = "GSPCA KONICA420"; break; case V4L2_PIX_FMT_MM21: descr = "Mediatek 8-bit Block Format"; break; case V4L2_PIX_FMT_HSV24: descr = "24-bit HSV 8-8-8"; break; case V4L2_PIX_FMT_HSV32: descr = "32-bit XHSV 8-8-8-8"; break; case V4L2_SDR_FMT_CU8: descr = "Complex U8"; break; case V4L2_SDR_FMT_CU16LE: descr = "Complex U16LE"; break; case V4L2_SDR_FMT_CS8: descr = "Complex S8"; break; case V4L2_SDR_FMT_CS14LE: descr = "Complex S14LE"; break; case V4L2_SDR_FMT_RU12LE: descr = "Real U12LE"; break; case V4L2_SDR_FMT_PCU16BE: descr = "Planar Complex U16BE"; break; case V4L2_SDR_FMT_PCU18BE: descr = "Planar Complex U18BE"; break; case V4L2_SDR_FMT_PCU20BE: descr = "Planar Complex U20BE"; break; case V4L2_TCH_FMT_DELTA_TD16: descr = "16-bit Signed Deltas"; break; case V4L2_TCH_FMT_DELTA_TD08: descr = "8-bit Signed Deltas"; break; case V4L2_TCH_FMT_TU16: descr = "16-bit Unsigned Touch Data"; break; case V4L2_TCH_FMT_TU08: descr = "8-bit Unsigned Touch Data"; break; case V4L2_META_FMT_VSP1_HGO: descr = "R-Car VSP1 1-D Histogram"; break; case V4L2_META_FMT_VSP1_HGT: descr = "R-Car VSP1 2-D Histogram"; break; case V4L2_META_FMT_UVC: descr = "UVC Payload Header Metadata"; break; case V4L2_META_FMT_D4XX: descr = "Intel D4xx UVC Metadata"; break; case V4L2_META_FMT_VIVID: descr = "Vivid Metadata"; break; case V4L2_META_FMT_RK_ISP1_PARAMS: descr = "Rockchip ISP1 3A Parameters"; break; case V4L2_META_FMT_RK_ISP1_STAT_3A: descr = "Rockchip ISP1 3A Statistics"; break; case V4L2_META_FMT_RK_ISP1_EXT_PARAMS: descr = "Rockchip ISP1 Ext 3A Params"; break; case V4L2_PIX_FMT_NV12_8L128: descr = "NV12 (8x128 Linear)"; break; case V4L2_PIX_FMT_NV12M_8L128: descr = "NV12M (8x128 Linear)"; break; case V4L2_PIX_FMT_NV12_10BE_8L128: descr = "10-bit NV12 (8x128 Linear, BE)"; break; case V4L2_PIX_FMT_NV12M_10BE_8L128: descr = "10-bit NV12M (8x128 Linear, BE)"; break; case V4L2_PIX_FMT_Y210: descr = "10-bit YUYV Packed"; break; case V4L2_PIX_FMT_Y212: descr = "12-bit YUYV Packed"; break; case V4L2_PIX_FMT_Y216: descr = "16-bit YUYV Packed"; break; case V4L2_META_FMT_RPI_BE_CFG: descr = "RPi PiSP BE Config format"; break; case V4L2_META_FMT_RPI_FE_CFG: descr = "RPi PiSP FE Config format"; break; case V4L2_META_FMT_RPI_FE_STATS: descr = "RPi PiSP FE Statistics format"; break; case V4L2_META_FMT_GENERIC_8: descr = "8-bit Generic Metadata"; break; case V4L2_META_FMT_GENERIC_CSI2_10: descr = "8-bit Generic Meta, 10b CSI-2"; break; case V4L2_META_FMT_GENERIC_CSI2_12: descr = "8-bit Generic Meta, 12b CSI-2"; break; case V4L2_META_FMT_GENERIC_CSI2_14: descr = "8-bit Generic Meta, 14b CSI-2"; break; case V4L2_META_FMT_GENERIC_CSI2_16: descr = "8-bit Generic Meta, 16b CSI-2"; break; case V4L2_META_FMT_GENERIC_CSI2_20: descr = "8-bit Generic Meta, 20b CSI-2"; break; case V4L2_META_FMT_GENERIC_CSI2_24: descr = "8-bit Generic Meta, 24b CSI-2"; break; default: /* Compressed formats */ flags = V4L2_FMT_FLAG_COMPRESSED; switch (fmt->pixelformat) { /* Max description length mask: descr = "0123456789012345678901234567890" */ case V4L2_PIX_FMT_MJPEG: descr = "Motion-JPEG"; break; case V4L2_PIX_FMT_JPEG: descr = "JFIF JPEG"; break; case V4L2_PIX_FMT_DV: descr = "1394"; break; case V4L2_PIX_FMT_MPEG: descr = "MPEG-1/2/4"; break; case V4L2_PIX_FMT_H264: descr = "H.264"; break; case V4L2_PIX_FMT_H264_NO_SC: descr = "H.264 (No Start Codes)"; break; case V4L2_PIX_FMT_H264_MVC: descr = "H.264 MVC"; break; case V4L2_PIX_FMT_H264_SLICE: descr = "H.264 Parsed Slice Data"; break; case V4L2_PIX_FMT_H263: descr = "H.263"; break; case V4L2_PIX_FMT_MPEG1: descr = "MPEG-1 ES"; break; case V4L2_PIX_FMT_MPEG2: descr = "MPEG-2 ES"; break; case V4L2_PIX_FMT_MPEG2_SLICE: descr = "MPEG-2 Parsed Slice Data"; break; case V4L2_PIX_FMT_MPEG4: descr = "MPEG-4 Part 2 ES"; break; case V4L2_PIX_FMT_XVID: descr = "Xvid"; break; case V4L2_PIX_FMT_VC1_ANNEX_G: descr = "VC-1 (SMPTE 412M Annex G)"; break; case V4L2_PIX_FMT_VC1_ANNEX_L: descr = "VC-1 (SMPTE 412M Annex L)"; break; case V4L2_PIX_FMT_VP8: descr = "VP8"; break; case V4L2_PIX_FMT_VP8_FRAME: descr = "VP8 Frame"; break; case V4L2_PIX_FMT_VP9: descr = "VP9"; break; case V4L2_PIX_FMT_VP9_FRAME: descr = "VP9 Frame"; break; case V4L2_PIX_FMT_HEVC: descr = "HEVC"; break; /* aka H.265 */ case V4L2_PIX_FMT_HEVC_SLICE: descr = "HEVC Parsed Slice Data"; break; case V4L2_PIX_FMT_FWHT: descr = "FWHT"; break; /* used in vicodec */ case V4L2_PIX_FMT_FWHT_STATELESS: descr = "FWHT Stateless"; break; /* used in vicodec */ case V4L2_PIX_FMT_SPK: descr = "Sorenson Spark"; break; case V4L2_PIX_FMT_RV30: descr = "RealVideo 8"; break; case V4L2_PIX_FMT_RV40: descr = "RealVideo 9 & 10"; break; case V4L2_PIX_FMT_CPIA1: descr = "GSPCA CPiA YUV"; break; case V4L2_PIX_FMT_WNVA: descr = "WNVA"; break; case V4L2_PIX_FMT_SN9C10X: descr = "GSPCA SN9C10X"; break; case V4L2_PIX_FMT_PWC1: descr = "Raw Philips Webcam Type (Old)"; break; case V4L2_PIX_FMT_PWC2: descr = "Raw Philips Webcam Type (New)"; break; case V4L2_PIX_FMT_ET61X251: descr = "GSPCA ET61X251"; break; case V4L2_PIX_FMT_SPCA561: descr = "GSPCA SPCA561"; break; case V4L2_PIX_FMT_PAC207: descr = "GSPCA PAC207"; break; case V4L2_PIX_FMT_MR97310A: descr = "GSPCA MR97310A"; break; case V4L2_PIX_FMT_JL2005BCD: descr = "GSPCA JL2005BCD"; break; case V4L2_PIX_FMT_SN9C2028: descr = "GSPCA SN9C2028"; break; case V4L2_PIX_FMT_SQ905C: descr = "GSPCA SQ905C"; break; case V4L2_PIX_FMT_PJPG: descr = "GSPCA PJPG"; break; case V4L2_PIX_FMT_OV511: descr = "GSPCA OV511"; break; case V4L2_PIX_FMT_OV518: descr = "GSPCA OV518"; break; case V4L2_PIX_FMT_JPGL: descr = "JPEG Lite"; break; case V4L2_PIX_FMT_SE401: descr = "GSPCA SE401"; break; case V4L2_PIX_FMT_S5C_UYVY_JPG: descr = "S5C73MX interleaved UYVY/JPEG"; break; case V4L2_PIX_FMT_MT21C: descr = "Mediatek Compressed Format"; break; case V4L2_PIX_FMT_QC08C: descr = "QCOM Compressed 8-bit Format"; break; case V4L2_PIX_FMT_QC10C: descr = "QCOM Compressed 10-bit Format"; break; case V4L2_PIX_FMT_AJPG: descr = "Aspeed JPEG"; break; case V4L2_PIX_FMT_AV1_FRAME: descr = "AV1 Frame"; break; case V4L2_PIX_FMT_MT2110T: descr = "Mediatek 10bit Tile Mode"; break; case V4L2_PIX_FMT_MT2110R: descr = "Mediatek 10bit Raster Mode"; break; case V4L2_PIX_FMT_HEXTILE: descr = "Hextile Compressed Format"; break; case V4L2_PIX_FMT_PISP_COMP1_RGGB: descr = "PiSP 8b RGRG/GBGB mode1 compr"; break; case V4L2_PIX_FMT_PISP_COMP1_GRBG: descr = "PiSP 8b GRGR/BGBG mode1 compr"; break; case V4L2_PIX_FMT_PISP_COMP1_GBRG: descr = "PiSP 8b GBGB/RGRG mode1 compr"; break; case V4L2_PIX_FMT_PISP_COMP1_BGGR: descr = "PiSP 8b BGBG/GRGR mode1 compr"; break; case V4L2_PIX_FMT_PISP_COMP1_MONO: descr = "PiSP 8b monochrome mode1 compr"; break; case V4L2_PIX_FMT_PISP_COMP2_RGGB: descr = "PiSP 8b RGRG/GBGB mode2 compr"; break; case V4L2_PIX_FMT_PISP_COMP2_GRBG: descr = "PiSP 8b GRGR/BGBG mode2 compr"; break; case V4L2_PIX_FMT_PISP_COMP2_GBRG: descr = "PiSP 8b GBGB/RGRG mode2 compr"; break; case V4L2_PIX_FMT_PISP_COMP2_BGGR: descr = "PiSP 8b BGBG/GRGR mode2 compr"; break; case V4L2_PIX_FMT_PISP_COMP2_MONO: descr = "PiSP 8b monochrome mode2 compr"; break; default: if (fmt->description[0]) return; WARN(1, "Unknown pixelformat 0x%08x\n", fmt->pixelformat); flags = 0; snprintf(fmt->description, sz, "%p4cc", &fmt->pixelformat); break; } } if (fmt->type == V4L2_BUF_TYPE_META_CAPTURE) { switch (fmt->pixelformat) { case V4L2_META_FMT_GENERIC_8: case V4L2_META_FMT_GENERIC_CSI2_10: case V4L2_META_FMT_GENERIC_CSI2_12: case V4L2_META_FMT_GENERIC_CSI2_14: case V4L2_META_FMT_GENERIC_CSI2_16: case V4L2_META_FMT_GENERIC_CSI2_20: case V4L2_META_FMT_GENERIC_CSI2_24: fmt->flags |= V4L2_FMT_FLAG_META_LINE_BASED; break; default: fmt->flags &= ~V4L2_FMT_FLAG_META_LINE_BASED; } } if (descr) WARN_ON(strscpy(fmt->description, descr, sz) < 0); fmt->flags |= flags; } static int v4l_enum_fmt(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vdev = video_devdata(file); struct v4l2_fmtdesc *p = arg; int ret = check_fmt(file, p->type); u32 mbus_code; u32 cap_mask; if (ret) return ret; ret = -EINVAL; if (!(vdev->device_caps & V4L2_CAP_IO_MC)) p->mbus_code = 0; mbus_code = p->mbus_code; memset_after(p, 0, type); p->mbus_code = mbus_code; switch (p->type) { case V4L2_BUF_TYPE_VIDEO_CAPTURE: case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: cap_mask = V4L2_CAP_VIDEO_CAPTURE_MPLANE | V4L2_CAP_VIDEO_M2M_MPLANE; if (!!(vdev->device_caps & cap_mask) != (p->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) break; if (unlikely(!ops->vidioc_enum_fmt_vid_cap)) break; ret = ops->vidioc_enum_fmt_vid_cap(file, fh, arg); break; case V4L2_BUF_TYPE_VIDEO_OVERLAY: if (unlikely(!ops->vidioc_enum_fmt_vid_overlay)) break; ret = ops->vidioc_enum_fmt_vid_overlay(file, fh, arg); break; case V4L2_BUF_TYPE_VIDEO_OUTPUT: case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: cap_mask = V4L2_CAP_VIDEO_OUTPUT_MPLANE | V4L2_CAP_VIDEO_M2M_MPLANE; if (!!(vdev->device_caps & cap_mask) != (p->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)) break; if (unlikely(!ops->vidioc_enum_fmt_vid_out)) break; ret = ops->vidioc_enum_fmt_vid_out(file, fh, arg); break; case V4L2_BUF_TYPE_SDR_CAPTURE: if (unlikely(!ops->vidioc_enum_fmt_sdr_cap)) break; ret = ops->vidioc_enum_fmt_sdr_cap(file, fh, arg); break; case V4L2_BUF_TYPE_SDR_OUTPUT: if (unlikely(!ops->vidioc_enum_fmt_sdr_out)) break; ret = ops->vidioc_enum_fmt_sdr_out(file, fh, arg); break; case V4L2_BUF_TYPE_META_CAPTURE: if (unlikely(!ops->vidioc_enum_fmt_meta_cap)) break; ret = ops->vidioc_enum_fmt_meta_cap(file, fh, arg); break; case V4L2_BUF_TYPE_META_OUTPUT: if (unlikely(!ops->vidioc_enum_fmt_meta_out)) break; ret = ops->vidioc_enum_fmt_meta_out(file, fh, arg); break; } if (ret == 0) v4l_fill_fmtdesc(p); return ret; } static void v4l_pix_format_touch(struct v4l2_pix_format *p) { /* * The v4l2_pix_format structure contains fields that make no sense for * touch. Set them to default values in this case. */ p->field = V4L2_FIELD_NONE; p->colorspace = V4L2_COLORSPACE_RAW; p->flags = 0; p->ycbcr_enc = 0; p->quantization = 0; p->xfer_func = 0; } static int v4l_g_fmt(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_format *p = arg; struct video_device *vfd = video_devdata(file); int ret = check_fmt(file, p->type); if (ret) return ret; memset(&p->fmt, 0, sizeof(p->fmt)); switch (p->type) { case V4L2_BUF_TYPE_VIDEO_CAPTURE: if (unlikely(!ops->vidioc_g_fmt_vid_cap)) break; p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC; ret = ops->vidioc_g_fmt_vid_cap(file, fh, arg); /* just in case the driver zeroed it again */ p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC; if (vfd->vfl_type == VFL_TYPE_TOUCH) v4l_pix_format_touch(&p->fmt.pix); return ret; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: return ops->vidioc_g_fmt_vid_cap_mplane(file, fh, arg); case V4L2_BUF_TYPE_VIDEO_OVERLAY: return ops->vidioc_g_fmt_vid_overlay(file, fh, arg); case V4L2_BUF_TYPE_VBI_CAPTURE: return ops->vidioc_g_fmt_vbi_cap(file, fh, arg); case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE: return ops->vidioc_g_fmt_sliced_vbi_cap(file, fh, arg); case V4L2_BUF_TYPE_VIDEO_OUTPUT: if (unlikely(!ops->vidioc_g_fmt_vid_out)) break; p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC; ret = ops->vidioc_g_fmt_vid_out(file, fh, arg); /* just in case the driver zeroed it again */ p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC; return ret; case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: return ops->vidioc_g_fmt_vid_out_mplane(file, fh, arg); case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY: return ops->vidioc_g_fmt_vid_out_overlay(file, fh, arg); case V4L2_BUF_TYPE_VBI_OUTPUT: return ops->vidioc_g_fmt_vbi_out(file, fh, arg); case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT: return ops->vidioc_g_fmt_sliced_vbi_out(file, fh, arg); case V4L2_BUF_TYPE_SDR_CAPTURE: return ops->vidioc_g_fmt_sdr_cap(file, fh, arg); case V4L2_BUF_TYPE_SDR_OUTPUT: return ops->vidioc_g_fmt_sdr_out(file, fh, arg); case V4L2_BUF_TYPE_META_CAPTURE: return ops->vidioc_g_fmt_meta_cap(file, fh, arg); case V4L2_BUF_TYPE_META_OUTPUT: return ops->vidioc_g_fmt_meta_out(file, fh, arg); } return -EINVAL; } static int v4l_s_fmt(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_format *p = arg; struct video_device *vfd = video_devdata(file); int ret = check_fmt(file, p->type); unsigned int i; if (ret) return ret; ret = v4l_enable_media_source(vfd); if (ret) return ret; v4l_sanitize_format(p); switch (p->type) { case V4L2_BUF_TYPE_VIDEO_CAPTURE: if (unlikely(!ops->vidioc_s_fmt_vid_cap)) break; memset_after(p, 0, fmt.pix); ret = ops->vidioc_s_fmt_vid_cap(file, fh, arg); /* just in case the driver zeroed it again */ p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC; if (vfd->vfl_type == VFL_TYPE_TOUCH) v4l_pix_format_touch(&p->fmt.pix); return ret; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: if (unlikely(!ops->vidioc_s_fmt_vid_cap_mplane)) break; memset_after(p, 0, fmt.pix_mp.xfer_func); for (i = 0; i < p->fmt.pix_mp.num_planes; i++) memset_after(&p->fmt.pix_mp.plane_fmt[i], 0, bytesperline); return ops->vidioc_s_fmt_vid_cap_mplane(file, fh, arg); case V4L2_BUF_TYPE_VIDEO_OVERLAY: if (unlikely(!ops->vidioc_s_fmt_vid_overlay)) break; memset_after(p, 0, fmt.win); p->fmt.win.clips = NULL; p->fmt.win.clipcount = 0; p->fmt.win.bitmap = NULL; return ops->vidioc_s_fmt_vid_overlay(file, fh, arg); case V4L2_BUF_TYPE_VBI_CAPTURE: if (unlikely(!ops->vidioc_s_fmt_vbi_cap)) break; memset_after(p, 0, fmt.vbi.flags); return ops->vidioc_s_fmt_vbi_cap(file, fh, arg); case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE: if (unlikely(!ops->vidioc_s_fmt_sliced_vbi_cap)) break; memset_after(p, 0, fmt.sliced.io_size); return ops->vidioc_s_fmt_sliced_vbi_cap(file, fh, arg); case V4L2_BUF_TYPE_VIDEO_OUTPUT: if (unlikely(!ops->vidioc_s_fmt_vid_out)) break; memset_after(p, 0, fmt.pix); ret = ops->vidioc_s_fmt_vid_out(file, fh, arg); /* just in case the driver zeroed it again */ p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC; return ret; case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: if (unlikely(!ops->vidioc_s_fmt_vid_out_mplane)) break; memset_after(p, 0, fmt.pix_mp.xfer_func); for (i = 0; i < p->fmt.pix_mp.num_planes; i++) memset_after(&p->fmt.pix_mp.plane_fmt[i], 0, bytesperline); return ops->vidioc_s_fmt_vid_out_mplane(file, fh, arg); case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY: if (unlikely(!ops->vidioc_s_fmt_vid_out_overlay)) break; memset_after(p, 0, fmt.win); p->fmt.win.clips = NULL; p->fmt.win.clipcount = 0; p->fmt.win.bitmap = NULL; return ops->vidioc_s_fmt_vid_out_overlay(file, fh, arg); case V4L2_BUF_TYPE_VBI_OUTPUT: if (unlikely(!ops->vidioc_s_fmt_vbi_out)) break; memset_after(p, 0, fmt.vbi.flags); return ops->vidioc_s_fmt_vbi_out(file, fh, arg); case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT: if (unlikely(!ops->vidioc_s_fmt_sliced_vbi_out)) break; memset_after(p, 0, fmt.sliced.io_size); return ops->vidioc_s_fmt_sliced_vbi_out(file, fh, arg); case V4L2_BUF_TYPE_SDR_CAPTURE: if (unlikely(!ops->vidioc_s_fmt_sdr_cap)) break; memset_after(p, 0, fmt.sdr.buffersize); return ops->vidioc_s_fmt_sdr_cap(file, fh, arg); case V4L2_BUF_TYPE_SDR_OUTPUT: if (unlikely(!ops->vidioc_s_fmt_sdr_out)) break; memset_after(p, 0, fmt.sdr.buffersize); return ops->vidioc_s_fmt_sdr_out(file, fh, arg); case V4L2_BUF_TYPE_META_CAPTURE: if (unlikely(!ops->vidioc_s_fmt_meta_cap)) break; memset_after(p, 0, fmt.meta); return ops->vidioc_s_fmt_meta_cap(file, fh, arg); case V4L2_BUF_TYPE_META_OUTPUT: if (unlikely(!ops->vidioc_s_fmt_meta_out)) break; memset_after(p, 0, fmt.meta); return ops->vidioc_s_fmt_meta_out(file, fh, arg); } return -EINVAL; } static int v4l_try_fmt(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_format *p = arg; struct video_device *vfd = video_devdata(file); int ret = check_fmt(file, p->type); unsigned int i; if (ret) return ret; v4l_sanitize_format(p); switch (p->type) { case V4L2_BUF_TYPE_VIDEO_CAPTURE: if (unlikely(!ops->vidioc_try_fmt_vid_cap)) break; memset_after(p, 0, fmt.pix); ret = ops->vidioc_try_fmt_vid_cap(file, fh, arg); /* just in case the driver zeroed it again */ p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC; if (vfd->vfl_type == VFL_TYPE_TOUCH) v4l_pix_format_touch(&p->fmt.pix); return ret; case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: if (unlikely(!ops->vidioc_try_fmt_vid_cap_mplane)) break; memset_after(p, 0, fmt.pix_mp.xfer_func); for (i = 0; i < p->fmt.pix_mp.num_planes; i++) memset_after(&p->fmt.pix_mp.plane_fmt[i], 0, bytesperline); return ops->vidioc_try_fmt_vid_cap_mplane(file, fh, arg); case V4L2_BUF_TYPE_VIDEO_OVERLAY: if (unlikely(!ops->vidioc_try_fmt_vid_overlay)) break; memset_after(p, 0, fmt.win); p->fmt.win.clips = NULL; p->fmt.win.clipcount = 0; p->fmt.win.bitmap = NULL; return ops->vidioc_try_fmt_vid_overlay(file, fh, arg); case V4L2_BUF_TYPE_VBI_CAPTURE: if (unlikely(!ops->vidioc_try_fmt_vbi_cap)) break; memset_after(p, 0, fmt.vbi.flags); return ops->vidioc_try_fmt_vbi_cap(file, fh, arg); case V4L2_BUF_TYPE_SLICED_VBI_CAPTURE: if (unlikely(!ops->vidioc_try_fmt_sliced_vbi_cap)) break; memset_after(p, 0, fmt.sliced.io_size); return ops->vidioc_try_fmt_sliced_vbi_cap(file, fh, arg); case V4L2_BUF_TYPE_VIDEO_OUTPUT: if (unlikely(!ops->vidioc_try_fmt_vid_out)) break; memset_after(p, 0, fmt.pix); ret = ops->vidioc_try_fmt_vid_out(file, fh, arg); /* just in case the driver zeroed it again */ p->fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC; return ret; case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: if (unlikely(!ops->vidioc_try_fmt_vid_out_mplane)) break; memset_after(p, 0, fmt.pix_mp.xfer_func); for (i = 0; i < p->fmt.pix_mp.num_planes; i++) memset_after(&p->fmt.pix_mp.plane_fmt[i], 0, bytesperline); return ops->vidioc_try_fmt_vid_out_mplane(file, fh, arg); case V4L2_BUF_TYPE_VIDEO_OUTPUT_OVERLAY: if (unlikely(!ops->vidioc_try_fmt_vid_out_overlay)) break; memset_after(p, 0, fmt.win); p->fmt.win.clips = NULL; p->fmt.win.clipcount = 0; p->fmt.win.bitmap = NULL; return ops->vidioc_try_fmt_vid_out_overlay(file, fh, arg); case V4L2_BUF_TYPE_VBI_OUTPUT: if (unlikely(!ops->vidioc_try_fmt_vbi_out)) break; memset_after(p, 0, fmt.vbi.flags); return ops->vidioc_try_fmt_vbi_out(file, fh, arg); case V4L2_BUF_TYPE_SLICED_VBI_OUTPUT: if (unlikely(!ops->vidioc_try_fmt_sliced_vbi_out)) break; memset_after(p, 0, fmt.sliced.io_size); return ops->vidioc_try_fmt_sliced_vbi_out(file, fh, arg); case V4L2_BUF_TYPE_SDR_CAPTURE: if (unlikely(!ops->vidioc_try_fmt_sdr_cap)) break; memset_after(p, 0, fmt.sdr.buffersize); return ops->vidioc_try_fmt_sdr_cap(file, fh, arg); case V4L2_BUF_TYPE_SDR_OUTPUT: if (unlikely(!ops->vidioc_try_fmt_sdr_out)) break; memset_after(p, 0, fmt.sdr.buffersize); return ops->vidioc_try_fmt_sdr_out(file, fh, arg); case V4L2_BUF_TYPE_META_CAPTURE: if (unlikely(!ops->vidioc_try_fmt_meta_cap)) break; memset_after(p, 0, fmt.meta); return ops->vidioc_try_fmt_meta_cap(file, fh, arg); case V4L2_BUF_TYPE_META_OUTPUT: if (unlikely(!ops->vidioc_try_fmt_meta_out)) break; memset_after(p, 0, fmt.meta); return ops->vidioc_try_fmt_meta_out(file, fh, arg); } return -EINVAL; } static int v4l_streamon(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { return ops->vidioc_streamon(file, fh, *(unsigned int *)arg); } static int v4l_streamoff(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { return ops->vidioc_streamoff(file, fh, *(unsigned int *)arg); } static int v4l_g_tuner(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_tuner *p = arg; int err; p->type = (vfd->vfl_type == VFL_TYPE_RADIO) ? V4L2_TUNER_RADIO : V4L2_TUNER_ANALOG_TV; err = ops->vidioc_g_tuner(file, fh, p); if (!err) p->capability |= V4L2_TUNER_CAP_FREQ_BANDS; return err; } static int v4l_s_tuner(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_tuner *p = arg; int ret; ret = v4l_enable_media_source(vfd); if (ret) return ret; p->type = (vfd->vfl_type == VFL_TYPE_RADIO) ? V4L2_TUNER_RADIO : V4L2_TUNER_ANALOG_TV; return ops->vidioc_s_tuner(file, fh, p); } static int v4l_g_modulator(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_modulator *p = arg; int err; if (vfd->vfl_type == VFL_TYPE_RADIO) p->type = V4L2_TUNER_RADIO; err = ops->vidioc_g_modulator(file, fh, p); if (!err) p->capability |= V4L2_TUNER_CAP_FREQ_BANDS; return err; } static int v4l_s_modulator(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_modulator *p = arg; if (vfd->vfl_type == VFL_TYPE_RADIO) p->type = V4L2_TUNER_RADIO; return ops->vidioc_s_modulator(file, fh, p); } static int v4l_g_frequency(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_frequency *p = arg; if (vfd->vfl_type == VFL_TYPE_SDR) p->type = V4L2_TUNER_SDR; else p->type = (vfd->vfl_type == VFL_TYPE_RADIO) ? V4L2_TUNER_RADIO : V4L2_TUNER_ANALOG_TV; return ops->vidioc_g_frequency(file, fh, p); } static int v4l_s_frequency(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); const struct v4l2_frequency *p = arg; enum v4l2_tuner_type type; int ret; ret = v4l_enable_media_source(vfd); if (ret) return ret; if (vfd->vfl_type == VFL_TYPE_SDR) { if (p->type != V4L2_TUNER_SDR && p->type != V4L2_TUNER_RF) return -EINVAL; } else { type = (vfd->vfl_type == VFL_TYPE_RADIO) ? V4L2_TUNER_RADIO : V4L2_TUNER_ANALOG_TV; if (type != p->type) return -EINVAL; } return ops->vidioc_s_frequency(file, fh, p); } static int v4l_enumstd(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_standard *p = arg; return v4l_video_std_enumstd(p, vfd->tvnorms); } static int v4l_s_std(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); v4l2_std_id id = *(v4l2_std_id *)arg, norm; int ret; ret = v4l_enable_media_source(vfd); if (ret) return ret; norm = id & vfd->tvnorms; if (vfd->tvnorms && !norm) /* Check if std is supported */ return -EINVAL; /* Calls the specific handler */ return ops->vidioc_s_std(file, fh, norm); } static int v4l_querystd(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); v4l2_std_id *p = arg; int ret; ret = v4l_enable_media_source(vfd); if (ret) return ret; /* * If no signal is detected, then the driver should return * V4L2_STD_UNKNOWN. Otherwise it should return tvnorms with * any standards that do not apply removed. * * This means that tuners, audio and video decoders can join * their efforts to improve the standards detection. */ *p = vfd->tvnorms; return ops->vidioc_querystd(file, fh, arg); } static int v4l_s_hw_freq_seek(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_hw_freq_seek *p = arg; enum v4l2_tuner_type type; int ret; ret = v4l_enable_media_source(vfd); if (ret) return ret; /* s_hw_freq_seek is not supported for SDR for now */ if (vfd->vfl_type == VFL_TYPE_SDR) return -EINVAL; type = (vfd->vfl_type == VFL_TYPE_RADIO) ? V4L2_TUNER_RADIO : V4L2_TUNER_ANALOG_TV; if (p->type != type) return -EINVAL; return ops->vidioc_s_hw_freq_seek(file, fh, p); } static int v4l_s_fbuf(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_framebuffer *p = arg; p->base = NULL; return ops->vidioc_s_fbuf(file, fh, p); } static int v4l_overlay(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { return ops->vidioc_overlay(file, fh, *(unsigned int *)arg); } static int v4l_reqbufs(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_requestbuffers *p = arg; int ret = check_fmt(file, p->type); if (ret) return ret; memset_after(p, 0, flags); p->capabilities = 0; if (is_valid_ioctl(vfd, VIDIOC_REMOVE_BUFS)) p->capabilities = V4L2_BUF_CAP_SUPPORTS_REMOVE_BUFS; return ops->vidioc_reqbufs(file, fh, p); } static int v4l_querybuf(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_buffer *p = arg; int ret = check_fmt(file, p->type); return ret ? ret : ops->vidioc_querybuf(file, fh, p); } static int v4l_qbuf(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_buffer *p = arg; int ret = check_fmt(file, p->type); return ret ? ret : ops->vidioc_qbuf(file, fh, p); } static int v4l_dqbuf(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_buffer *p = arg; int ret = check_fmt(file, p->type); return ret ? ret : ops->vidioc_dqbuf(file, fh, p); } static int v4l_create_bufs(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_create_buffers *create = arg; int ret = check_fmt(file, create->format.type); if (ret) return ret; memset_after(create, 0, flags); v4l_sanitize_format(&create->format); create->capabilities = 0; if (is_valid_ioctl(vfd, VIDIOC_REMOVE_BUFS)) create->capabilities = V4L2_BUF_CAP_SUPPORTS_REMOVE_BUFS; ret = ops->vidioc_create_bufs(file, fh, create); if (create->format.type == V4L2_BUF_TYPE_VIDEO_CAPTURE || create->format.type == V4L2_BUF_TYPE_VIDEO_OUTPUT) create->format.fmt.pix.priv = V4L2_PIX_FMT_PRIV_MAGIC; return ret; } static int v4l_prepare_buf(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_buffer *b = arg; int ret = check_fmt(file, b->type); return ret ? ret : ops->vidioc_prepare_buf(file, fh, b); } static int v4l_remove_bufs(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_remove_buffers *remove = arg; if (ops->vidioc_remove_bufs) return ops->vidioc_remove_bufs(file, fh, remove); return -ENOTTY; } static int v4l_g_parm(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_streamparm *p = arg; v4l2_std_id std; int ret = check_fmt(file, p->type); if (ret) return ret; if (ops->vidioc_g_parm) return ops->vidioc_g_parm(file, fh, p); if (p->type != V4L2_BUF_TYPE_VIDEO_CAPTURE && p->type != V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) return -EINVAL; if (vfd->device_caps & V4L2_CAP_READWRITE) p->parm.capture.readbuffers = 2; ret = ops->vidioc_g_std(file, fh, &std); if (ret == 0) v4l2_video_std_frame_period(std, &p->parm.capture.timeperframe); return ret; } static int v4l_s_parm(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_streamparm *p = arg; int ret = check_fmt(file, p->type); if (ret) return ret; /* Note: extendedmode is never used in drivers */ if (V4L2_TYPE_IS_OUTPUT(p->type)) { memset(p->parm.output.reserved, 0, sizeof(p->parm.output.reserved)); p->parm.output.extendedmode = 0; p->parm.output.outputmode &= V4L2_MODE_HIGHQUALITY; } else { memset(p->parm.capture.reserved, 0, sizeof(p->parm.capture.reserved)); p->parm.capture.extendedmode = 0; p->parm.capture.capturemode &= V4L2_MODE_HIGHQUALITY; } return ops->vidioc_s_parm(file, fh, p); } static int v4l_queryctrl(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_queryctrl *p = arg; struct v4l2_fh *vfh = test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags) ? fh : NULL; if (vfh && vfh->ctrl_handler) return v4l2_queryctrl(vfh->ctrl_handler, p); if (vfd->ctrl_handler) return v4l2_queryctrl(vfd->ctrl_handler, p); if (ops->vidioc_queryctrl) return ops->vidioc_queryctrl(file, fh, p); return -ENOTTY; } static int v4l_query_ext_ctrl(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_query_ext_ctrl *p = arg; struct v4l2_fh *vfh = test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags) ? fh : NULL; if (vfh && vfh->ctrl_handler) return v4l2_query_ext_ctrl(vfh->ctrl_handler, p); if (vfd->ctrl_handler) return v4l2_query_ext_ctrl(vfd->ctrl_handler, p); if (ops->vidioc_query_ext_ctrl) return ops->vidioc_query_ext_ctrl(file, fh, p); return -ENOTTY; } static int v4l_querymenu(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_querymenu *p = arg; struct v4l2_fh *vfh = test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags) ? fh : NULL; if (vfh && vfh->ctrl_handler) return v4l2_querymenu(vfh->ctrl_handler, p); if (vfd->ctrl_handler) return v4l2_querymenu(vfd->ctrl_handler, p); if (ops->vidioc_querymenu) return ops->vidioc_querymenu(file, fh, p); return -ENOTTY; } static int v4l_g_ctrl(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_control *p = arg; struct v4l2_fh *vfh = test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags) ? fh : NULL; struct v4l2_ext_controls ctrls; struct v4l2_ext_control ctrl; if (vfh && vfh->ctrl_handler) return v4l2_g_ctrl(vfh->ctrl_handler, p); if (vfd->ctrl_handler) return v4l2_g_ctrl(vfd->ctrl_handler, p); if (ops->vidioc_g_ctrl) return ops->vidioc_g_ctrl(file, fh, p); if (ops->vidioc_g_ext_ctrls == NULL) return -ENOTTY; ctrls.which = V4L2_CTRL_ID2WHICH(p->id); ctrls.count = 1; ctrls.controls = &ctrl; ctrl.id = p->id; ctrl.value = p->value; if (check_ext_ctrls(&ctrls, VIDIOC_G_CTRL)) { int ret = ops->vidioc_g_ext_ctrls(file, fh, &ctrls); if (ret == 0) p->value = ctrl.value; return ret; } return -EINVAL; } static int v4l_s_ctrl(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_control *p = arg; struct v4l2_fh *vfh = test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags) ? fh : NULL; struct v4l2_ext_controls ctrls; struct v4l2_ext_control ctrl; int ret; if (vfh && vfh->ctrl_handler) return v4l2_s_ctrl(vfh, vfh->ctrl_handler, p); if (vfd->ctrl_handler) return v4l2_s_ctrl(NULL, vfd->ctrl_handler, p); if (ops->vidioc_s_ctrl) return ops->vidioc_s_ctrl(file, fh, p); if (ops->vidioc_s_ext_ctrls == NULL) return -ENOTTY; ctrls.which = V4L2_CTRL_ID2WHICH(p->id); ctrls.count = 1; ctrls.controls = &ctrl; ctrl.id = p->id; ctrl.value = p->value; if (!check_ext_ctrls(&ctrls, VIDIOC_S_CTRL)) return -EINVAL; ret = ops->vidioc_s_ext_ctrls(file, fh, &ctrls); p->value = ctrl.value; return ret; } static int v4l_g_ext_ctrls(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_ext_controls *p = arg; struct v4l2_fh *vfh = test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags) ? fh : NULL; p->error_idx = p->count; if (vfh && vfh->ctrl_handler) return v4l2_g_ext_ctrls(vfh->ctrl_handler, vfd, vfd->v4l2_dev->mdev, p); if (vfd->ctrl_handler) return v4l2_g_ext_ctrls(vfd->ctrl_handler, vfd, vfd->v4l2_dev->mdev, p); if (ops->vidioc_g_ext_ctrls == NULL) return -ENOTTY; return check_ext_ctrls(p, VIDIOC_G_EXT_CTRLS) ? ops->vidioc_g_ext_ctrls(file, fh, p) : -EINVAL; } static int v4l_s_ext_ctrls(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_ext_controls *p = arg; struct v4l2_fh *vfh = test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags) ? fh : NULL; p->error_idx = p->count; if (vfh && vfh->ctrl_handler) return v4l2_s_ext_ctrls(vfh, vfh->ctrl_handler, vfd, vfd->v4l2_dev->mdev, p); if (vfd->ctrl_handler) return v4l2_s_ext_ctrls(NULL, vfd->ctrl_handler, vfd, vfd->v4l2_dev->mdev, p); if (ops->vidioc_s_ext_ctrls == NULL) return -ENOTTY; return check_ext_ctrls(p, VIDIOC_S_EXT_CTRLS) ? ops->vidioc_s_ext_ctrls(file, fh, p) : -EINVAL; } static int v4l_try_ext_ctrls(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_ext_controls *p = arg; struct v4l2_fh *vfh = test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags) ? fh : NULL; p->error_idx = p->count; if (vfh && vfh->ctrl_handler) return v4l2_try_ext_ctrls(vfh->ctrl_handler, vfd, vfd->v4l2_dev->mdev, p); if (vfd->ctrl_handler) return v4l2_try_ext_ctrls(vfd->ctrl_handler, vfd, vfd->v4l2_dev->mdev, p); if (ops->vidioc_try_ext_ctrls == NULL) return -ENOTTY; return check_ext_ctrls(p, VIDIOC_TRY_EXT_CTRLS) ? ops->vidioc_try_ext_ctrls(file, fh, p) : -EINVAL; } /* * The selection API specified originally that the _MPLANE buffer types * shouldn't be used. The reasons for this are lost in the mists of time * (or just really crappy memories). Regardless, this is really annoying * for userspace. So to keep things simple we map _MPLANE buffer types * to their 'regular' counterparts before calling the driver. And we * restore it afterwards. This way applications can use either buffer * type and drivers don't need to check for both. */ static int v4l_g_selection(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_selection *p = arg; u32 old_type = p->type; int ret; if (p->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) p->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; else if (p->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) p->type = V4L2_BUF_TYPE_VIDEO_OUTPUT; ret = ops->vidioc_g_selection(file, fh, p); p->type = old_type; return ret; } static int v4l_s_selection(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_selection *p = arg; u32 old_type = p->type; int ret; if (p->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) p->type = V4L2_BUF_TYPE_VIDEO_CAPTURE; else if (p->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) p->type = V4L2_BUF_TYPE_VIDEO_OUTPUT; ret = ops->vidioc_s_selection(file, fh, p); p->type = old_type; return ret; } static int v4l_g_crop(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_crop *p = arg; struct v4l2_selection s = { .type = p->type, }; int ret; /* simulate capture crop using selection api */ /* crop means compose for output devices */ if (V4L2_TYPE_IS_OUTPUT(p->type)) s.target = V4L2_SEL_TGT_COMPOSE; else s.target = V4L2_SEL_TGT_CROP; if (test_bit(V4L2_FL_QUIRK_INVERTED_CROP, &vfd->flags)) s.target = s.target == V4L2_SEL_TGT_COMPOSE ? V4L2_SEL_TGT_CROP : V4L2_SEL_TGT_COMPOSE; ret = v4l_g_selection(ops, file, fh, &s); /* copying results to old structure on success */ if (!ret) p->c = s.r; return ret; } static int v4l_s_crop(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_crop *p = arg; struct v4l2_selection s = { .type = p->type, .r = p->c, }; /* simulate capture crop using selection api */ /* crop means compose for output devices */ if (V4L2_TYPE_IS_OUTPUT(p->type)) s.target = V4L2_SEL_TGT_COMPOSE; else s.target = V4L2_SEL_TGT_CROP; if (test_bit(V4L2_FL_QUIRK_INVERTED_CROP, &vfd->flags)) s.target = s.target == V4L2_SEL_TGT_COMPOSE ? V4L2_SEL_TGT_CROP : V4L2_SEL_TGT_COMPOSE; return v4l_s_selection(ops, file, fh, &s); } static int v4l_cropcap(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_cropcap *p = arg; struct v4l2_selection s = { .type = p->type }; int ret = 0; /* setting trivial pixelaspect */ p->pixelaspect.numerator = 1; p->pixelaspect.denominator = 1; if (s.type == V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE) s.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; else if (s.type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) s.type = V4L2_BUF_TYPE_VIDEO_OUTPUT; /* * The determine_valid_ioctls() call already should ensure * that this can never happen, but just in case... */ if (WARN_ON(!ops->vidioc_g_selection)) return -ENOTTY; if (ops->vidioc_g_pixelaspect) ret = ops->vidioc_g_pixelaspect(file, fh, s.type, &p->pixelaspect); /* * Ignore ENOTTY or ENOIOCTLCMD error returns, just use the * square pixel aspect ratio in that case. */ if (ret && ret != -ENOTTY && ret != -ENOIOCTLCMD) return ret; /* Use g_selection() to fill in the bounds and defrect rectangles */ /* obtaining bounds */ if (V4L2_TYPE_IS_OUTPUT(p->type)) s.target = V4L2_SEL_TGT_COMPOSE_BOUNDS; else s.target = V4L2_SEL_TGT_CROP_BOUNDS; if (test_bit(V4L2_FL_QUIRK_INVERTED_CROP, &vfd->flags)) s.target = s.target == V4L2_SEL_TGT_COMPOSE_BOUNDS ? V4L2_SEL_TGT_CROP_BOUNDS : V4L2_SEL_TGT_COMPOSE_BOUNDS; ret = v4l_g_selection(ops, file, fh, &s); if (ret) return ret; p->bounds = s.r; /* obtaining defrect */ if (s.target == V4L2_SEL_TGT_COMPOSE_BOUNDS) s.target = V4L2_SEL_TGT_COMPOSE_DEFAULT; else s.target = V4L2_SEL_TGT_CROP_DEFAULT; ret = v4l_g_selection(ops, file, fh, &s); if (ret) return ret; p->defrect = s.r; return 0; } static int v4l_log_status(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); int ret; if (vfd->v4l2_dev) pr_info("%s: ================= START STATUS =================\n", vfd->v4l2_dev->name); ret = ops->vidioc_log_status(file, fh); if (vfd->v4l2_dev) pr_info("%s: ================== END STATUS ==================\n", vfd->v4l2_dev->name); return ret; } static int v4l_dbg_g_register(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { #ifdef CONFIG_VIDEO_ADV_DEBUG struct v4l2_dbg_register *p = arg; struct video_device *vfd = video_devdata(file); struct v4l2_subdev *sd; int idx = 0; if (!capable(CAP_SYS_ADMIN)) return -EPERM; if (p->match.type == V4L2_CHIP_MATCH_SUBDEV) { if (vfd->v4l2_dev == NULL) return -EINVAL; v4l2_device_for_each_subdev(sd, vfd->v4l2_dev) if (p->match.addr == idx++) return v4l2_subdev_call(sd, core, g_register, p); return -EINVAL; } if (ops->vidioc_g_register && p->match.type == V4L2_CHIP_MATCH_BRIDGE && (ops->vidioc_g_chip_info || p->match.addr == 0)) return ops->vidioc_g_register(file, fh, p); return -EINVAL; #else return -ENOTTY; #endif } static int v4l_dbg_s_register(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { #ifdef CONFIG_VIDEO_ADV_DEBUG const struct v4l2_dbg_register *p = arg; struct video_device *vfd = video_devdata(file); struct v4l2_subdev *sd; int idx = 0; if (!capable(CAP_SYS_ADMIN)) return -EPERM; if (p->match.type == V4L2_CHIP_MATCH_SUBDEV) { if (vfd->v4l2_dev == NULL) return -EINVAL; v4l2_device_for_each_subdev(sd, vfd->v4l2_dev) if (p->match.addr == idx++) return v4l2_subdev_call(sd, core, s_register, p); return -EINVAL; } if (ops->vidioc_s_register && p->match.type == V4L2_CHIP_MATCH_BRIDGE && (ops->vidioc_g_chip_info || p->match.addr == 0)) return ops->vidioc_s_register(file, fh, p); return -EINVAL; #else return -ENOTTY; #endif } static int v4l_dbg_g_chip_info(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { #ifdef CONFIG_VIDEO_ADV_DEBUG struct video_device *vfd = video_devdata(file); struct v4l2_dbg_chip_info *p = arg; struct v4l2_subdev *sd; int idx = 0; switch (p->match.type) { case V4L2_CHIP_MATCH_BRIDGE: if (ops->vidioc_s_register) p->flags |= V4L2_CHIP_FL_WRITABLE; if (ops->vidioc_g_register) p->flags |= V4L2_CHIP_FL_READABLE; strscpy(p->name, vfd->v4l2_dev->name, sizeof(p->name)); if (ops->vidioc_g_chip_info) return ops->vidioc_g_chip_info(file, fh, arg); if (p->match.addr) return -EINVAL; return 0; case V4L2_CHIP_MATCH_SUBDEV: if (vfd->v4l2_dev == NULL) break; v4l2_device_for_each_subdev(sd, vfd->v4l2_dev) { if (p->match.addr != idx++) continue; if (sd->ops->core && sd->ops->core->s_register) p->flags |= V4L2_CHIP_FL_WRITABLE; if (sd->ops->core && sd->ops->core->g_register) p->flags |= V4L2_CHIP_FL_READABLE; strscpy(p->name, sd->name, sizeof(p->name)); return 0; } break; } return -EINVAL; #else return -ENOTTY; #endif } static int v4l_dqevent(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { return v4l2_event_dequeue(fh, arg, file->f_flags & O_NONBLOCK); } static int v4l_subscribe_event(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { return ops->vidioc_subscribe_event(fh, arg); } static int v4l_unsubscribe_event(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { return ops->vidioc_unsubscribe_event(fh, arg); } static int v4l_g_sliced_vbi_cap(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct v4l2_sliced_vbi_cap *p = arg; int ret = check_fmt(file, p->type); if (ret) return ret; /* Clear up to type, everything after type is zeroed already */ memset(p, 0, offsetof(struct v4l2_sliced_vbi_cap, type)); return ops->vidioc_g_sliced_vbi_cap(file, fh, p); } static int v4l_enum_freq_bands(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *arg) { struct video_device *vfd = video_devdata(file); struct v4l2_frequency_band *p = arg; enum v4l2_tuner_type type; int err; if (vfd->vfl_type == VFL_TYPE_SDR) { if (p->type != V4L2_TUNER_SDR && p->type != V4L2_TUNER_RF) return -EINVAL; type = p->type; } else { type = (vfd->vfl_type == VFL_TYPE_RADIO) ? V4L2_TUNER_RADIO : V4L2_TUNER_ANALOG_TV; if (type != p->type) return -EINVAL; } if (ops->vidioc_enum_freq_bands) { err = ops->vidioc_enum_freq_bands(file, fh, p); if (err != -ENOTTY) return err; } if (is_valid_ioctl(vfd, VIDIOC_G_TUNER)) { struct v4l2_tuner t = { .index = p->tuner, .type = type, }; if (p->index) return -EINVAL; err = ops->vidioc_g_tuner(file, fh, &t); if (err) return err; p->capability = t.capability | V4L2_TUNER_CAP_FREQ_BANDS; p->rangelow = t.rangelow; p->rangehigh = t.rangehigh; p->modulation = (type == V4L2_TUNER_RADIO) ? V4L2_BAND_MODULATION_FM : V4L2_BAND_MODULATION_VSB; return 0; } if (is_valid_ioctl(vfd, VIDIOC_G_MODULATOR)) { struct v4l2_modulator m = { .index = p->tuner, }; if (type != V4L2_TUNER_RADIO) return -EINVAL; if (p->index) return -EINVAL; err = ops->vidioc_g_modulator(file, fh, &m); if (err) return err; p->capability = m.capability | V4L2_TUNER_CAP_FREQ_BANDS; p->rangelow = m.rangelow; p->rangehigh = m.rangehigh; p->modulation = (type == V4L2_TUNER_RADIO) ? V4L2_BAND_MODULATION_FM : V4L2_BAND_MODULATION_VSB; return 0; } return -ENOTTY; } struct v4l2_ioctl_info { unsigned int ioctl; u32 flags; const char * const name; int (*func)(const struct v4l2_ioctl_ops *ops, struct file *file, void *fh, void *p); void (*debug)(const void *arg, bool write_only); }; /* This control needs a priority check */ #define INFO_FL_PRIO (1 << 0) /* This control can be valid if the filehandle passes a control handler. */ #define INFO_FL_CTRL (1 << 1) /* Queuing ioctl */ #define INFO_FL_QUEUE (1 << 2) /* Always copy back result, even on error */ #define INFO_FL_ALWAYS_COPY (1 << 3) /* Zero struct from after the field to the end */ #define INFO_FL_CLEAR(v4l2_struct, field) \ ((offsetof(struct v4l2_struct, field) + \ sizeof_field(struct v4l2_struct, field)) << 16) #define INFO_FL_CLEAR_MASK (_IOC_SIZEMASK << 16) #define DEFINE_V4L_STUB_FUNC(_vidioc) \ static int v4l_stub_ ## _vidioc( \ const struct v4l2_ioctl_ops *ops, \ struct file *file, void *fh, void *p) \ { \ return ops->vidioc_ ## _vidioc(file, fh, p); \ } #define IOCTL_INFO(_ioctl, _func, _debug, _flags) \ [_IOC_NR(_ioctl)] = { \ .ioctl = _ioctl, \ .flags = _flags, \ .name = #_ioctl, \ .func = _func, \ .debug = _debug, \ } DEFINE_V4L_STUB_FUNC(g_fbuf) DEFINE_V4L_STUB_FUNC(expbuf) DEFINE_V4L_STUB_FUNC(g_std) DEFINE_V4L_STUB_FUNC(g_audio) DEFINE_V4L_STUB_FUNC(s_audio) DEFINE_V4L_STUB_FUNC(g_edid) DEFINE_V4L_STUB_FUNC(s_edid) DEFINE_V4L_STUB_FUNC(g_audout) DEFINE_V4L_STUB_FUNC(s_audout) DEFINE_V4L_STUB_FUNC(g_jpegcomp) DEFINE_V4L_STUB_FUNC(s_jpegcomp) DEFINE_V4L_STUB_FUNC(enumaudio) DEFINE_V4L_STUB_FUNC(enumaudout) DEFINE_V4L_STUB_FUNC(enum_framesizes) DEFINE_V4L_STUB_FUNC(enum_frameintervals) DEFINE_V4L_STUB_FUNC(g_enc_index) DEFINE_V4L_STUB_FUNC(encoder_cmd) DEFINE_V4L_STUB_FUNC(try_encoder_cmd) DEFINE_V4L_STUB_FUNC(decoder_cmd) DEFINE_V4L_STUB_FUNC(try_decoder_cmd) DEFINE_V4L_STUB_FUNC(s_dv_timings) DEFINE_V4L_STUB_FUNC(g_dv_timings) DEFINE_V4L_STUB_FUNC(enum_dv_timings) DEFINE_V4L_STUB_FUNC(query_dv_timings) DEFINE_V4L_STUB_FUNC(dv_timings_cap) static const struct v4l2_ioctl_info v4l2_ioctls[] = { IOCTL_INFO(VIDIOC_QUERYCAP, v4l_querycap, v4l_print_querycap, 0), IOCTL_INFO(VIDIOC_ENUM_FMT, v4l_enum_fmt, v4l_print_fmtdesc, 0), IOCTL_INFO(VIDIOC_G_FMT, v4l_g_fmt, v4l_print_format, 0), IOCTL_INFO(VIDIOC_S_FMT, v4l_s_fmt, v4l_print_format, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_REQBUFS, v4l_reqbufs, v4l_print_requestbuffers, INFO_FL_PRIO | INFO_FL_QUEUE), IOCTL_INFO(VIDIOC_QUERYBUF, v4l_querybuf, v4l_print_buffer, INFO_FL_QUEUE | INFO_FL_CLEAR(v4l2_buffer, length)), IOCTL_INFO(VIDIOC_G_FBUF, v4l_stub_g_fbuf, v4l_print_framebuffer, 0), IOCTL_INFO(VIDIOC_S_FBUF, v4l_s_fbuf, v4l_print_framebuffer, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_OVERLAY, v4l_overlay, v4l_print_u32, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_QBUF, v4l_qbuf, v4l_print_buffer, INFO_FL_QUEUE), IOCTL_INFO(VIDIOC_EXPBUF, v4l_stub_expbuf, v4l_print_exportbuffer, INFO_FL_QUEUE | INFO_FL_CLEAR(v4l2_exportbuffer, flags)), IOCTL_INFO(VIDIOC_DQBUF, v4l_dqbuf, v4l_print_buffer, INFO_FL_QUEUE), IOCTL_INFO(VIDIOC_STREAMON, v4l_streamon, v4l_print_buftype, INFO_FL_PRIO | INFO_FL_QUEUE), IOCTL_INFO(VIDIOC_STREAMOFF, v4l_streamoff, v4l_print_buftype, INFO_FL_PRIO | INFO_FL_QUEUE), IOCTL_INFO(VIDIOC_G_PARM, v4l_g_parm, v4l_print_streamparm, INFO_FL_CLEAR(v4l2_streamparm, type)), IOCTL_INFO(VIDIOC_S_PARM, v4l_s_parm, v4l_print_streamparm, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_G_STD, v4l_stub_g_std, v4l_print_std, 0), IOCTL_INFO(VIDIOC_S_STD, v4l_s_std, v4l_print_std, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_ENUMSTD, v4l_enumstd, v4l_print_standard, INFO_FL_CLEAR(v4l2_standard, index)), IOCTL_INFO(VIDIOC_ENUMINPUT, v4l_enuminput, v4l_print_enuminput, INFO_FL_CLEAR(v4l2_input, index)), IOCTL_INFO(VIDIOC_G_CTRL, v4l_g_ctrl, v4l_print_control, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_control, id)), IOCTL_INFO(VIDIOC_S_CTRL, v4l_s_ctrl, v4l_print_control, INFO_FL_PRIO | INFO_FL_CTRL), IOCTL_INFO(VIDIOC_G_TUNER, v4l_g_tuner, v4l_print_tuner, INFO_FL_CLEAR(v4l2_tuner, index)), IOCTL_INFO(VIDIOC_S_TUNER, v4l_s_tuner, v4l_print_tuner, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_G_AUDIO, v4l_stub_g_audio, v4l_print_audio, 0), IOCTL_INFO(VIDIOC_S_AUDIO, v4l_stub_s_audio, v4l_print_audio, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_QUERYCTRL, v4l_queryctrl, v4l_print_queryctrl, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_queryctrl, id)), IOCTL_INFO(VIDIOC_QUERYMENU, v4l_querymenu, v4l_print_querymenu, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_querymenu, index)), IOCTL_INFO(VIDIOC_G_INPUT, v4l_g_input, v4l_print_u32, 0), IOCTL_INFO(VIDIOC_S_INPUT, v4l_s_input, v4l_print_u32, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_G_EDID, v4l_stub_g_edid, v4l_print_edid, INFO_FL_ALWAYS_COPY), IOCTL_INFO(VIDIOC_S_EDID, v4l_stub_s_edid, v4l_print_edid, INFO_FL_PRIO | INFO_FL_ALWAYS_COPY), IOCTL_INFO(VIDIOC_G_OUTPUT, v4l_g_output, v4l_print_u32, 0), IOCTL_INFO(VIDIOC_S_OUTPUT, v4l_s_output, v4l_print_u32, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_ENUMOUTPUT, v4l_enumoutput, v4l_print_enumoutput, INFO_FL_CLEAR(v4l2_output, index)), IOCTL_INFO(VIDIOC_G_AUDOUT, v4l_stub_g_audout, v4l_print_audioout, 0), IOCTL_INFO(VIDIOC_S_AUDOUT, v4l_stub_s_audout, v4l_print_audioout, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_G_MODULATOR, v4l_g_modulator, v4l_print_modulator, INFO_FL_CLEAR(v4l2_modulator, index)), IOCTL_INFO(VIDIOC_S_MODULATOR, v4l_s_modulator, v4l_print_modulator, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_G_FREQUENCY, v4l_g_frequency, v4l_print_frequency, INFO_FL_CLEAR(v4l2_frequency, tuner)), IOCTL_INFO(VIDIOC_S_FREQUENCY, v4l_s_frequency, v4l_print_frequency, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_CROPCAP, v4l_cropcap, v4l_print_cropcap, INFO_FL_CLEAR(v4l2_cropcap, type)), IOCTL_INFO(VIDIOC_G_CROP, v4l_g_crop, v4l_print_crop, INFO_FL_CLEAR(v4l2_crop, type)), IOCTL_INFO(VIDIOC_S_CROP, v4l_s_crop, v4l_print_crop, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_G_SELECTION, v4l_g_selection, v4l_print_selection, INFO_FL_CLEAR(v4l2_selection, r)), IOCTL_INFO(VIDIOC_S_SELECTION, v4l_s_selection, v4l_print_selection, INFO_FL_PRIO | INFO_FL_CLEAR(v4l2_selection, r)), IOCTL_INFO(VIDIOC_G_JPEGCOMP, v4l_stub_g_jpegcomp, v4l_print_jpegcompression, 0), IOCTL_INFO(VIDIOC_S_JPEGCOMP, v4l_stub_s_jpegcomp, v4l_print_jpegcompression, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_QUERYSTD, v4l_querystd, v4l_print_std, 0), IOCTL_INFO(VIDIOC_TRY_FMT, v4l_try_fmt, v4l_print_format, 0), IOCTL_INFO(VIDIOC_ENUMAUDIO, v4l_stub_enumaudio, v4l_print_audio, INFO_FL_CLEAR(v4l2_audio, index)), IOCTL_INFO(VIDIOC_ENUMAUDOUT, v4l_stub_enumaudout, v4l_print_audioout, INFO_FL_CLEAR(v4l2_audioout, index)), IOCTL_INFO(VIDIOC_G_PRIORITY, v4l_g_priority, v4l_print_u32, 0), IOCTL_INFO(VIDIOC_S_PRIORITY, v4l_s_priority, v4l_print_u32, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_G_SLICED_VBI_CAP, v4l_g_sliced_vbi_cap, v4l_print_sliced_vbi_cap, INFO_FL_CLEAR(v4l2_sliced_vbi_cap, type)), IOCTL_INFO(VIDIOC_LOG_STATUS, v4l_log_status, v4l_print_newline, 0), IOCTL_INFO(VIDIOC_G_EXT_CTRLS, v4l_g_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL | INFO_FL_ALWAYS_COPY), IOCTL_INFO(VIDIOC_S_EXT_CTRLS, v4l_s_ext_ctrls, v4l_print_ext_controls, INFO_FL_PRIO | INFO_FL_CTRL | INFO_FL_ALWAYS_COPY), IOCTL_INFO(VIDIOC_TRY_EXT_CTRLS, v4l_try_ext_ctrls, v4l_print_ext_controls, INFO_FL_CTRL | INFO_FL_ALWAYS_COPY), IOCTL_INFO(VIDIOC_ENUM_FRAMESIZES, v4l_stub_enum_framesizes, v4l_print_frmsizeenum, INFO_FL_CLEAR(v4l2_frmsizeenum, pixel_format)), IOCTL_INFO(VIDIOC_ENUM_FRAMEINTERVALS, v4l_stub_enum_frameintervals, v4l_print_frmivalenum, INFO_FL_CLEAR(v4l2_frmivalenum, height)), IOCTL_INFO(VIDIOC_G_ENC_INDEX, v4l_stub_g_enc_index, v4l_print_enc_idx, 0), IOCTL_INFO(VIDIOC_ENCODER_CMD, v4l_stub_encoder_cmd, v4l_print_encoder_cmd, INFO_FL_PRIO | INFO_FL_CLEAR(v4l2_encoder_cmd, flags)), IOCTL_INFO(VIDIOC_TRY_ENCODER_CMD, v4l_stub_try_encoder_cmd, v4l_print_encoder_cmd, INFO_FL_CLEAR(v4l2_encoder_cmd, flags)), IOCTL_INFO(VIDIOC_DECODER_CMD, v4l_stub_decoder_cmd, v4l_print_decoder_cmd, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_TRY_DECODER_CMD, v4l_stub_try_decoder_cmd, v4l_print_decoder_cmd, 0), IOCTL_INFO(VIDIOC_DBG_S_REGISTER, v4l_dbg_s_register, v4l_print_dbg_register, 0), IOCTL_INFO(VIDIOC_DBG_G_REGISTER, v4l_dbg_g_register, v4l_print_dbg_register, 0), IOCTL_INFO(VIDIOC_S_HW_FREQ_SEEK, v4l_s_hw_freq_seek, v4l_print_hw_freq_seek, INFO_FL_PRIO), IOCTL_INFO(VIDIOC_S_DV_TIMINGS, v4l_stub_s_dv_timings, v4l_print_dv_timings, INFO_FL_PRIO | INFO_FL_CLEAR(v4l2_dv_timings, bt.flags)), IOCTL_INFO(VIDIOC_G_DV_TIMINGS, v4l_stub_g_dv_timings, v4l_print_dv_timings, 0), IOCTL_INFO(VIDIOC_DQEVENT, v4l_dqevent, v4l_print_event, 0), IOCTL_INFO(VIDIOC_SUBSCRIBE_EVENT, v4l_subscribe_event, v4l_print_event_subscription, 0), IOCTL_INFO(VIDIOC_UNSUBSCRIBE_EVENT, v4l_unsubscribe_event, v4l_print_event_subscription, 0), IOCTL_INFO(VIDIOC_CREATE_BUFS, v4l_create_bufs, v4l_print_create_buffers, INFO_FL_PRIO | INFO_FL_QUEUE), IOCTL_INFO(VIDIOC_PREPARE_BUF, v4l_prepare_buf, v4l_print_buffer, INFO_FL_QUEUE), IOCTL_INFO(VIDIOC_ENUM_DV_TIMINGS, v4l_stub_enum_dv_timings, v4l_print_enum_dv_timings, INFO_FL_CLEAR(v4l2_enum_dv_timings, pad)), IOCTL_INFO(VIDIOC_QUERY_DV_TIMINGS, v4l_stub_query_dv_timings, v4l_print_dv_timings, INFO_FL_ALWAYS_COPY), IOCTL_INFO(VIDIOC_DV_TIMINGS_CAP, v4l_stub_dv_timings_cap, v4l_print_dv_timings_cap, INFO_FL_CLEAR(v4l2_dv_timings_cap, pad)), IOCTL_INFO(VIDIOC_ENUM_FREQ_BANDS, v4l_enum_freq_bands, v4l_print_freq_band, 0), IOCTL_INFO(VIDIOC_DBG_G_CHIP_INFO, v4l_dbg_g_chip_info, v4l_print_dbg_chip_info, INFO_FL_CLEAR(v4l2_dbg_chip_info, match)), IOCTL_INFO(VIDIOC_QUERY_EXT_CTRL, v4l_query_ext_ctrl, v4l_print_query_ext_ctrl, INFO_FL_CTRL | INFO_FL_CLEAR(v4l2_query_ext_ctrl, id)), IOCTL_INFO(VIDIOC_REMOVE_BUFS, v4l_remove_bufs, v4l_print_remove_buffers, INFO_FL_PRIO | INFO_FL_QUEUE | INFO_FL_CLEAR(v4l2_remove_buffers, type)), }; #define V4L2_IOCTLS ARRAY_SIZE(v4l2_ioctls) static bool v4l2_is_known_ioctl(unsigned int cmd) { if (_IOC_NR(cmd) >= V4L2_IOCTLS) return false; return v4l2_ioctls[_IOC_NR(cmd)].ioctl == cmd; } static struct mutex *v4l2_ioctl_get_lock(struct video_device *vdev, struct v4l2_fh *vfh, unsigned int cmd, void *arg) { if (_IOC_NR(cmd) >= V4L2_IOCTLS) return vdev->lock; if (vfh && vfh->m2m_ctx && (v4l2_ioctls[_IOC_NR(cmd)].flags & INFO_FL_QUEUE)) { if (vfh->m2m_ctx->q_lock) return vfh->m2m_ctx->q_lock; } if (vdev->queue && vdev->queue->lock && (v4l2_ioctls[_IOC_NR(cmd)].flags & INFO_FL_QUEUE)) return vdev->queue->lock; return vdev->lock; } /* Common ioctl debug function. This function can be used by external ioctl messages as well as internal V4L ioctl */ void v4l_printk_ioctl(const char *prefix, unsigned int cmd) { const char *dir, *type; if (prefix) printk(KERN_DEBUG "%s: ", prefix); switch (_IOC_TYPE(cmd)) { case 'd': type = "v4l2_int"; break; case 'V': if (!v4l2_is_known_ioctl(cmd)) { type = "v4l2"; break; } pr_cont("%s", v4l2_ioctls[_IOC_NR(cmd)].name); return; default: type = "unknown"; break; } switch (_IOC_DIR(cmd)) { case _IOC_NONE: dir = "--"; break; case _IOC_READ: dir = "r-"; break; case _IOC_WRITE: dir = "-w"; break; case _IOC_READ | _IOC_WRITE: dir = "rw"; break; default: dir = "*ERR*"; break; } pr_cont("%s ioctl '%c', dir=%s, #%d (0x%08x)", type, _IOC_TYPE(cmd), dir, _IOC_NR(cmd), cmd); } EXPORT_SYMBOL(v4l_printk_ioctl); static long __video_do_ioctl(struct file *file, unsigned int cmd, void *arg) { struct video_device *vfd = video_devdata(file); struct mutex *req_queue_lock = NULL; struct mutex *lock; /* ioctl serialization mutex */ const struct v4l2_ioctl_ops *ops = vfd->ioctl_ops; bool write_only = false; struct v4l2_ioctl_info default_info; const struct v4l2_ioctl_info *info; void *fh = file->private_data; struct v4l2_fh *vfh = NULL; int dev_debug = vfd->dev_debug; long ret = -ENOTTY; if (ops == NULL) { pr_warn("%s: has no ioctl_ops.\n", video_device_node_name(vfd)); return ret; } if (test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags)) vfh = file->private_data; /* * We need to serialize streamon/off with queueing new requests. * These ioctls may trigger the cancellation of a streaming * operation, and that should not be mixed with queueing a new * request at the same time. */ if (v4l2_device_supports_requests(vfd->v4l2_dev) && (cmd == VIDIOC_STREAMON || cmd == VIDIOC_STREAMOFF)) { req_queue_lock = &vfd->v4l2_dev->mdev->req_queue_mutex; if (mutex_lock_interruptible(req_queue_lock)) return -ERESTARTSYS; } lock = v4l2_ioctl_get_lock(vfd, vfh, cmd, arg); if (lock && mutex_lock_interruptible(lock)) { if (req_queue_lock) mutex_unlock(req_queue_lock); return -ERESTARTSYS; } if (!video_is_registered(vfd)) { ret = -ENODEV; goto unlock; } if (v4l2_is_known_ioctl(cmd)) { info = &v4l2_ioctls[_IOC_NR(cmd)]; if (!is_valid_ioctl(vfd, cmd) && !((info->flags & INFO_FL_CTRL) && vfh && vfh->ctrl_handler)) goto done; if (vfh && (info->flags & INFO_FL_PRIO)) { ret = v4l2_prio_check(vfd->prio, vfh->prio); if (ret) goto done; } } else { default_info.ioctl = cmd; default_info.flags = 0; default_info.debug = v4l_print_default; info = &default_info; } write_only = _IOC_DIR(cmd) == _IOC_WRITE; if (info != &default_info) { ret = info->func(ops, file, fh, arg); } else if (!ops->vidioc_default) { ret = -ENOTTY; } else { ret = ops->vidioc_default(file, fh, vfh ? v4l2_prio_check(vfd->prio, vfh->prio) >= 0 : 0, cmd, arg); } done: if (dev_debug & (V4L2_DEV_DEBUG_IOCTL | V4L2_DEV_DEBUG_IOCTL_ARG)) { if (!(dev_debug & V4L2_DEV_DEBUG_STREAMING) && (cmd == VIDIOC_QBUF || cmd == VIDIOC_DQBUF)) goto unlock; v4l_printk_ioctl(video_device_node_name(vfd), cmd); if (ret < 0) pr_cont(": error %ld", ret); if (!(dev_debug & V4L2_DEV_DEBUG_IOCTL_ARG)) pr_cont("\n"); else if (_IOC_DIR(cmd) == _IOC_NONE) info->debug(arg, write_only); else { pr_cont(": "); info->debug(arg, write_only); } } unlock: if (lock) mutex_unlock(lock); if (req_queue_lock) mutex_unlock(req_queue_lock); return ret; } static int check_array_args(unsigned int cmd, void *parg, size_t *array_size, void __user **user_ptr, void ***kernel_ptr) { int ret = 0; switch (cmd) { case VIDIOC_PREPARE_BUF: case VIDIOC_QUERYBUF: case VIDIOC_QBUF: case VIDIOC_DQBUF: { struct v4l2_buffer *buf = parg; if (V4L2_TYPE_IS_MULTIPLANAR(buf->type) && buf->length > 0) { if (buf->length > VIDEO_MAX_PLANES) { ret = -EINVAL; break; } *user_ptr = (void __user *)buf->m.planes; *kernel_ptr = (void **)&buf->m.planes; *array_size = sizeof(struct v4l2_plane) * buf->length; ret = 1; } break; } case VIDIOC_G_EDID: case VIDIOC_S_EDID: { struct v4l2_edid *edid = parg; if (edid->blocks) { if (edid->blocks > 256) { ret = -EINVAL; break; } *user_ptr = (void __user *)edid->edid; *kernel_ptr = (void **)&edid->edid; *array_size = edid->blocks * 128; ret = 1; } break; } case VIDIOC_S_EXT_CTRLS: case VIDIOC_G_EXT_CTRLS: case VIDIOC_TRY_EXT_CTRLS: { struct v4l2_ext_controls *ctrls = parg; if (ctrls->count != 0) { if (ctrls->count > V4L2_CID_MAX_CTRLS) { ret = -EINVAL; break; } *user_ptr = (void __user *)ctrls->controls; *kernel_ptr = (void **)&ctrls->controls; *array_size = sizeof(struct v4l2_ext_control) * ctrls->count; ret = 1; } break; } case VIDIOC_SUBDEV_G_ROUTING: case VIDIOC_SUBDEV_S_ROUTING: { struct v4l2_subdev_routing *routing = parg; if (routing->len_routes > 256) return -E2BIG; *user_ptr = u64_to_user_ptr(routing->routes); *kernel_ptr = (void **)&routing->routes; *array_size = sizeof(struct v4l2_subdev_route) * routing->len_routes; ret = 1; break; } } return ret; } static unsigned int video_translate_cmd(unsigned int cmd) { #if !defined(CONFIG_64BIT) && defined(CONFIG_COMPAT_32BIT_TIME) switch (cmd) { case VIDIOC_DQEVENT_TIME32: return VIDIOC_DQEVENT; case VIDIOC_QUERYBUF_TIME32: return VIDIOC_QUERYBUF; case VIDIOC_QBUF_TIME32: return VIDIOC_QBUF; case VIDIOC_DQBUF_TIME32: return VIDIOC_DQBUF; case VIDIOC_PREPARE_BUF_TIME32: return VIDIOC_PREPARE_BUF; } #endif if (in_compat_syscall()) return v4l2_compat_translate_cmd(cmd); return cmd; } static int video_get_user(void __user *arg, void *parg, unsigned int real_cmd, unsigned int cmd, bool *always_copy) { unsigned int n = _IOC_SIZE(real_cmd); int err = 0; if (!(_IOC_DIR(cmd) & _IOC_WRITE)) { /* read-only ioctl */ memset(parg, 0, n); return 0; } /* * In some cases, only a few fields are used as input, * i.e. when the app sets "index" and then the driver * fills in the rest of the structure for the thing * with that index. We only need to copy up the first * non-input field. */ if (v4l2_is_known_ioctl(real_cmd)) { u32 flags = v4l2_ioctls[_IOC_NR(real_cmd)].flags; if (flags & INFO_FL_CLEAR_MASK) n = (flags & INFO_FL_CLEAR_MASK) >> 16; *always_copy = flags & INFO_FL_ALWAYS_COPY; } if (cmd == real_cmd) { if (copy_from_user(parg, (void __user *)arg, n)) err = -EFAULT; } else if (in_compat_syscall()) { memset(parg, 0, n); err = v4l2_compat_get_user(arg, parg, cmd); } else { memset(parg, 0, n); #if !defined(CONFIG_64BIT) && defined(CONFIG_COMPAT_32BIT_TIME) switch (cmd) { case VIDIOC_QUERYBUF_TIME32: case VIDIOC_QBUF_TIME32: case VIDIOC_DQBUF_TIME32: case VIDIOC_PREPARE_BUF_TIME32: { struct v4l2_buffer_time32 vb32; struct v4l2_buffer *vb = parg; if (copy_from_user(&vb32, arg, sizeof(vb32))) return -EFAULT; *vb = (struct v4l2_buffer) { .index = vb32.index, .type = vb32.type, .bytesused = vb32.bytesused, .flags = vb32.flags, .field = vb32.field, .timestamp.tv_sec = vb32.timestamp.tv_sec, .timestamp.tv_usec = vb32.timestamp.tv_usec, .timecode = vb32.timecode, .sequence = vb32.sequence, .memory = vb32.memory, .m.userptr = vb32.m.userptr, .length = vb32.length, .request_fd = vb32.request_fd, }; break; } } #endif } /* zero out anything we don't copy from userspace */ if (!err && n < _IOC_SIZE(real_cmd)) memset((u8 *)parg + n, 0, _IOC_SIZE(real_cmd) - n); return err; } static int video_put_user(void __user *arg, void *parg, unsigned int real_cmd, unsigned int cmd) { if (!(_IOC_DIR(cmd) & _IOC_READ)) return 0; if (cmd == real_cmd) { /* Copy results into user buffer */ if (copy_to_user(arg, parg, _IOC_SIZE(cmd))) return -EFAULT; return 0; } if (in_compat_syscall()) return v4l2_compat_put_user(arg, parg, cmd); #if !defined(CONFIG_64BIT) && defined(CONFIG_COMPAT_32BIT_TIME) switch (cmd) { case VIDIOC_DQEVENT_TIME32: { struct v4l2_event *ev = parg; struct v4l2_event_time32 ev32; memset(&ev32, 0, sizeof(ev32)); ev32.type = ev->type; ev32.pending = ev->pending; ev32.sequence = ev->sequence; ev32.timestamp.tv_sec = ev->timestamp.tv_sec; ev32.timestamp.tv_nsec = ev->timestamp.tv_nsec; ev32.id = ev->id; memcpy(&ev32.u, &ev->u, sizeof(ev->u)); memcpy(&ev32.reserved, &ev->reserved, sizeof(ev->reserved)); if (copy_to_user(arg, &ev32, sizeof(ev32))) return -EFAULT; break; } case VIDIOC_QUERYBUF_TIME32: case VIDIOC_QBUF_TIME32: case VIDIOC_DQBUF_TIME32: case VIDIOC_PREPARE_BUF_TIME32: { struct v4l2_buffer *vb = parg; struct v4l2_buffer_time32 vb32; memset(&vb32, 0, sizeof(vb32)); vb32.index = vb->index; vb32.type = vb->type; vb32.bytesused = vb->bytesused; vb32.flags = vb->flags; vb32.field = vb->field; vb32.timestamp.tv_sec = vb->timestamp.tv_sec; vb32.timestamp.tv_usec = vb->timestamp.tv_usec; vb32.timecode = vb->timecode; vb32.sequence = vb->sequence; vb32.memory = vb->memory; vb32.m.userptr = vb->m.userptr; vb32.length = vb->length; vb32.request_fd = vb->request_fd; if (copy_to_user(arg, &vb32, sizeof(vb32))) return -EFAULT; break; } } #endif return 0; } long video_usercopy(struct file *file, unsigned int orig_cmd, unsigned long arg, v4l2_kioctl func) { char sbuf[128]; void *mbuf = NULL, *array_buf = NULL; void *parg = (void *)arg; long err = -EINVAL; bool has_array_args; bool always_copy = false; size_t array_size = 0; void __user *user_ptr = NULL; void **kernel_ptr = NULL; unsigned int cmd = video_translate_cmd(orig_cmd); const size_t ioc_size = _IOC_SIZE(cmd); /* Copy arguments into temp kernel buffer */ if (_IOC_DIR(cmd) != _IOC_NONE) { if (ioc_size <= sizeof(sbuf)) { parg = sbuf; } else { /* too big to allocate from stack */ mbuf = kmalloc(ioc_size, GFP_KERNEL); if (NULL == mbuf) return -ENOMEM; parg = mbuf; } err = video_get_user((void __user *)arg, parg, cmd, orig_cmd, &always_copy); if (err) goto out; } err = check_array_args(cmd, parg, &array_size, &user_ptr, &kernel_ptr); if (err < 0) goto out; has_array_args = err; if (has_array_args) { array_buf = kvmalloc(array_size, GFP_KERNEL); err = -ENOMEM; if (array_buf == NULL) goto out; if (in_compat_syscall()) err = v4l2_compat_get_array_args(file, array_buf, user_ptr, array_size, orig_cmd, parg); else err = copy_from_user(array_buf, user_ptr, array_size) ? -EFAULT : 0; if (err) goto out; *kernel_ptr = array_buf; } /* Handles IOCTL */ err = func(file, cmd, parg); if (err == -ENOTTY || err == -ENOIOCTLCMD) { err = -ENOTTY; goto out; } if (err == 0) { if (cmd == VIDIOC_DQBUF) trace_v4l2_dqbuf(video_devdata(file)->minor, parg); else if (cmd == VIDIOC_QBUF) trace_v4l2_qbuf(video_devdata(file)->minor, parg); } /* * Some ioctls can return an error, but still have valid * results that must be returned. * * FIXME: subdev IOCTLS are partially handled here and partially in * v4l2-subdev.c and the 'always_copy' flag can only be set for IOCTLS * defined here as part of the 'v4l2_ioctls' array. As * VIDIOC_SUBDEV_[GS]_ROUTING needs to return results to applications * even in case of failure, but it is not defined here as part of the * 'v4l2_ioctls' array, insert an ad-hoc check to address that. */ if (cmd == VIDIOC_SUBDEV_G_ROUTING || cmd == VIDIOC_SUBDEV_S_ROUTING) always_copy = true; if (err < 0 && !always_copy) goto out; if (has_array_args) { *kernel_ptr = (void __force *)user_ptr; if (in_compat_syscall()) { int put_err; put_err = v4l2_compat_put_array_args(file, user_ptr, array_buf, array_size, orig_cmd, parg); if (put_err) err = put_err; } else if (copy_to_user(user_ptr, array_buf, array_size)) { err = -EFAULT; } } if (video_put_user((void __user *)arg, parg, cmd, orig_cmd)) err = -EFAULT; out: kvfree(array_buf); kfree(mbuf); return err; } long video_ioctl2(struct file *file, unsigned int cmd, unsigned long arg) { return video_usercopy(file, cmd, arg, __video_do_ioctl); } EXPORT_SYMBOL(video_ioctl2); |
| 6 1 2 2 1 2 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 | /* * Davicom DM96xx USB 10/100Mbps ethernet devices * * Peter Korsgaard <jacmet@sunsite.dk> * * This file is licensed under the terms of the GNU General Public License * version 2. This program is licensed "as is" without any warranty of any * kind, whether express or implied. */ //#define DEBUG #include <linux/module.h> #include <linux/sched.h> #include <linux/stddef.h> #include <linux/netdevice.h> #include <linux/etherdevice.h> #include <linux/ethtool.h> #include <linux/mii.h> #include <linux/usb.h> #include <linux/crc32.h> #include <linux/usb/usbnet.h> #include <linux/slab.h> /* datasheet: http://ptm2.cc.utu.fi/ftp/network/cards/DM9601/From_NET/DM9601-DS-P01-930914.pdf */ /* control requests */ #define DM_READ_REGS 0x00 #define DM_WRITE_REGS 0x01 #define DM_READ_MEMS 0x02 #define DM_WRITE_REG 0x03 #define DM_WRITE_MEMS 0x05 #define DM_WRITE_MEM 0x07 /* registers */ #define DM_NET_CTRL 0x00 #define DM_RX_CTRL 0x05 #define DM_SHARED_CTRL 0x0b #define DM_SHARED_ADDR 0x0c #define DM_SHARED_DATA 0x0d /* low + high */ #define DM_PHY_ADDR 0x10 /* 6 bytes */ #define DM_MCAST_ADDR 0x16 /* 8 bytes */ #define DM_GPR_CTRL 0x1e #define DM_GPR_DATA 0x1f #define DM_CHIP_ID 0x2c #define DM_MODE_CTRL 0x91 /* only on dm9620 */ /* chip id values */ #define ID_DM9601 0 #define ID_DM9620 1 #define DM_MAX_MCAST 64 #define DM_MCAST_SIZE 8 #define DM_EEPROM_LEN 256 #define DM_TX_OVERHEAD 2 /* 2 byte header */ #define DM_RX_OVERHEAD 7 /* 3 byte header + 4 byte crc tail */ #define DM_TIMEOUT 1000 static int dm_read(struct usbnet *dev, u8 reg, u16 length, void *data) { int err; err = usbnet_read_cmd(dev, DM_READ_REGS, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, 0, reg, data, length); if(err != length && err >= 0) err = -EINVAL; return err; } static int dm_read_reg(struct usbnet *dev, u8 reg, u8 *value) { return dm_read(dev, reg, 1, value); } static int dm_write(struct usbnet *dev, u8 reg, u16 length, void *data) { int err; err = usbnet_write_cmd(dev, DM_WRITE_REGS, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE, 0, reg, data, length); if (err >= 0 && err < length) err = -EINVAL; return err; } static int dm_write_reg(struct usbnet *dev, u8 reg, u8 value) { return usbnet_write_cmd(dev, DM_WRITE_REG, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE, value, reg, NULL, 0); } static void dm_write_async(struct usbnet *dev, u8 reg, u16 length, const void *data) { usbnet_write_cmd_async(dev, DM_WRITE_REGS, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE, 0, reg, data, length); } static void dm_write_reg_async(struct usbnet *dev, u8 reg, u8 value) { usbnet_write_cmd_async(dev, DM_WRITE_REG, USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE, value, reg, NULL, 0); } static int dm_read_shared_word(struct usbnet *dev, int phy, u8 reg, __le16 *value) { int ret, i; mutex_lock(&dev->phy_mutex); dm_write_reg(dev, DM_SHARED_ADDR, phy ? (reg | 0x40) : reg); dm_write_reg(dev, DM_SHARED_CTRL, phy ? 0xc : 0x4); for (i = 0; i < DM_TIMEOUT; i++) { u8 tmp = 0; udelay(1); ret = dm_read_reg(dev, DM_SHARED_CTRL, &tmp); if (ret < 0) goto out; /* ready */ if ((tmp & 1) == 0) break; } if (i == DM_TIMEOUT) { netdev_err(dev->net, "%s read timed out!\n", phy ? "phy" : "eeprom"); ret = -EIO; goto out; } dm_write_reg(dev, DM_SHARED_CTRL, 0x0); ret = dm_read(dev, DM_SHARED_DATA, 2, value); netdev_dbg(dev->net, "read shared %d 0x%02x returned 0x%04x, %d\n", phy, reg, *value, ret); out: mutex_unlock(&dev->phy_mutex); return ret; } static int dm_write_shared_word(struct usbnet *dev, int phy, u8 reg, __le16 value) { int ret, i; mutex_lock(&dev->phy_mutex); ret = dm_write(dev, DM_SHARED_DATA, 2, &value); if (ret < 0) goto out; dm_write_reg(dev, DM_SHARED_ADDR, phy ? (reg | 0x40) : reg); dm_write_reg(dev, DM_SHARED_CTRL, phy ? 0x1a : 0x12); for (i = 0; i < DM_TIMEOUT; i++) { u8 tmp = 0; udelay(1); ret = dm_read_reg(dev, DM_SHARED_CTRL, &tmp); if (ret < 0) goto out; /* ready */ if ((tmp & 1) == 0) break; } if (i == DM_TIMEOUT) { netdev_err(dev->net, "%s write timed out!\n", phy ? "phy" : "eeprom"); ret = -EIO; goto out; } dm_write_reg(dev, DM_SHARED_CTRL, 0x0); out: mutex_unlock(&dev->phy_mutex); return ret; } static int dm_read_eeprom_word(struct usbnet *dev, u8 offset, void *value) { return dm_read_shared_word(dev, 0, offset, value); } static int dm9601_get_eeprom_len(struct net_device *dev) { return DM_EEPROM_LEN; } static int dm9601_get_eeprom(struct net_device *net, struct ethtool_eeprom *eeprom, u8 * data) { struct usbnet *dev = netdev_priv(net); __le16 *ebuf = (__le16 *) data; int i; /* access is 16bit */ if ((eeprom->offset % 2) || (eeprom->len % 2)) return -EINVAL; for (i = 0; i < eeprom->len / 2; i++) { if (dm_read_eeprom_word(dev, eeprom->offset / 2 + i, &ebuf[i]) < 0) return -EINVAL; } return 0; } static int dm9601_mdio_read(struct net_device *netdev, int phy_id, int loc) { struct usbnet *dev = netdev_priv(netdev); __le16 res; int err; if (phy_id) { netdev_dbg(dev->net, "Only internal phy supported\n"); return 0; } err = dm_read_shared_word(dev, 1, loc, &res); if (err < 0) { netdev_err(dev->net, "MDIO read error: %d\n", err); return 0; } netdev_dbg(dev->net, "dm9601_mdio_read() phy_id=0x%02x, loc=0x%02x, returns=0x%04x\n", phy_id, loc, le16_to_cpu(res)); return le16_to_cpu(res); } static void dm9601_mdio_write(struct net_device *netdev, int phy_id, int loc, int val) { struct usbnet *dev = netdev_priv(netdev); __le16 res = cpu_to_le16(val); if (phy_id) { netdev_dbg(dev->net, "Only internal phy supported\n"); return; } netdev_dbg(dev->net, "dm9601_mdio_write() phy_id=0x%02x, loc=0x%02x, val=0x%04x\n", phy_id, loc, val); dm_write_shared_word(dev, 1, loc, res); } static void dm9601_get_drvinfo(struct net_device *net, struct ethtool_drvinfo *info) { /* Inherit standard device info */ usbnet_get_drvinfo(net, info); } static u32 dm9601_get_link(struct net_device *net) { struct usbnet *dev = netdev_priv(net); return mii_link_ok(&dev->mii); } static int dm9601_ioctl(struct net_device *net, struct ifreq *rq, int cmd) { struct usbnet *dev = netdev_priv(net); return generic_mii_ioctl(&dev->mii, if_mii(rq), cmd, NULL); } static const struct ethtool_ops dm9601_ethtool_ops = { .get_drvinfo = dm9601_get_drvinfo, .get_link = dm9601_get_link, .get_msglevel = usbnet_get_msglevel, .set_msglevel = usbnet_set_msglevel, .get_eeprom_len = dm9601_get_eeprom_len, .get_eeprom = dm9601_get_eeprom, .nway_reset = usbnet_nway_reset, .get_link_ksettings = usbnet_get_link_ksettings_mii, .set_link_ksettings = usbnet_set_link_ksettings_mii, }; static void dm9601_set_multicast(struct net_device *net) { struct usbnet *dev = netdev_priv(net); /* We use the 20 byte dev->data for our 8 byte filter buffer * to avoid allocating memory that is tricky to free later */ u8 *hashes = (u8 *) & dev->data; u8 rx_ctl = 0x31; memset(hashes, 0x00, DM_MCAST_SIZE); hashes[DM_MCAST_SIZE - 1] |= 0x80; /* broadcast address */ if (net->flags & IFF_PROMISC) { rx_ctl |= 0x02; } else if (net->flags & IFF_ALLMULTI || netdev_mc_count(net) > DM_MAX_MCAST) { rx_ctl |= 0x08; } else if (!netdev_mc_empty(net)) { struct netdev_hw_addr *ha; netdev_for_each_mc_addr(ha, net) { u32 crc = ether_crc(ETH_ALEN, ha->addr) >> 26; hashes[crc >> 3] |= 1 << (crc & 0x7); } } dm_write_async(dev, DM_MCAST_ADDR, DM_MCAST_SIZE, hashes); dm_write_reg_async(dev, DM_RX_CTRL, rx_ctl); } static void __dm9601_set_mac_address(struct usbnet *dev) { dm_write_async(dev, DM_PHY_ADDR, ETH_ALEN, dev->net->dev_addr); } static int dm9601_set_mac_address(struct net_device *net, void *p) { struct sockaddr *addr = p; struct usbnet *dev = netdev_priv(net); if (!is_valid_ether_addr(addr->sa_data)) { dev_err(&net->dev, "not setting invalid mac address %pM\n", addr->sa_data); return -EINVAL; } eth_hw_addr_set(net, addr->sa_data); __dm9601_set_mac_address(dev); return 0; } static const struct net_device_ops dm9601_netdev_ops = { .ndo_open = usbnet_open, .ndo_stop = usbnet_stop, .ndo_start_xmit = usbnet_start_xmit, .ndo_tx_timeout = usbnet_tx_timeout, .ndo_change_mtu = usbnet_change_mtu, .ndo_get_stats64 = dev_get_tstats64, .ndo_validate_addr = eth_validate_addr, .ndo_eth_ioctl = dm9601_ioctl, .ndo_set_rx_mode = dm9601_set_multicast, .ndo_set_mac_address = dm9601_set_mac_address, }; static int dm9601_bind(struct usbnet *dev, struct usb_interface *intf) { int ret; u8 mac[ETH_ALEN], id; ret = usbnet_get_endpoints(dev, intf); if (ret) goto out; dev->net->netdev_ops = &dm9601_netdev_ops; dev->net->ethtool_ops = &dm9601_ethtool_ops; dev->net->hard_header_len += DM_TX_OVERHEAD; dev->hard_mtu = dev->net->mtu + dev->net->hard_header_len; /* dm9620/21a require room for 4 byte padding, even in dm9601 * mode, so we need +1 to be able to receive full size * ethernet frames. */ dev->rx_urb_size = dev->net->mtu + ETH_HLEN + DM_RX_OVERHEAD + 1; dev->mii.dev = dev->net; dev->mii.mdio_read = dm9601_mdio_read; dev->mii.mdio_write = dm9601_mdio_write; dev->mii.phy_id_mask = 0x1f; dev->mii.reg_num_mask = 0x1f; /* reset */ dm_write_reg(dev, DM_NET_CTRL, 1); udelay(20); /* read MAC */ if (dm_read(dev, DM_PHY_ADDR, ETH_ALEN, mac) < 0) { printk(KERN_ERR "Error reading MAC address\n"); ret = -ENODEV; goto out; } /* * Overwrite the auto-generated address only with good ones. */ if (is_valid_ether_addr(mac)) eth_hw_addr_set(dev->net, mac); else { printk(KERN_WARNING "dm9601: No valid MAC address in EEPROM, using %pM\n", dev->net->dev_addr); __dm9601_set_mac_address(dev); } if (dm_read_reg(dev, DM_CHIP_ID, &id) < 0) { netdev_err(dev->net, "Error reading chip ID\n"); ret = -ENODEV; goto out; } /* put dm9620 devices in dm9601 mode */ if (id == ID_DM9620) { u8 mode; if (dm_read_reg(dev, DM_MODE_CTRL, &mode) < 0) { netdev_err(dev->net, "Error reading MODE_CTRL\n"); ret = -ENODEV; goto out; } dm_write_reg(dev, DM_MODE_CTRL, mode & 0x7f); } /* power up phy */ dm_write_reg(dev, DM_GPR_CTRL, 1); dm_write_reg(dev, DM_GPR_DATA, 0); /* receive broadcast packets */ dm9601_set_multicast(dev->net); dm9601_mdio_write(dev->net, dev->mii.phy_id, MII_BMCR, BMCR_RESET); dm9601_mdio_write(dev->net, dev->mii.phy_id, MII_ADVERTISE, ADVERTISE_ALL | ADVERTISE_CSMA | ADVERTISE_PAUSE_CAP); mii_nway_restart(&dev->mii); out: return ret; } static int dm9601_rx_fixup(struct usbnet *dev, struct sk_buff *skb) { u8 status; int len; /* format: b1: rx status b2: packet length (incl crc) low b3: packet length (incl crc) high b4..n-4: packet data bn-3..bn: ethernet crc */ if (unlikely(skb->len < DM_RX_OVERHEAD)) { dev_err(&dev->udev->dev, "unexpected tiny rx frame\n"); return 0; } status = skb->data[0]; len = (skb->data[1] | (skb->data[2] << 8)) - 4; if (unlikely(status & 0xbf)) { if (status & 0x01) dev->net->stats.rx_fifo_errors++; if (status & 0x02) dev->net->stats.rx_crc_errors++; if (status & 0x04) dev->net->stats.rx_frame_errors++; if (status & 0x20) dev->net->stats.rx_missed_errors++; if (status & 0x90) dev->net->stats.rx_length_errors++; return 0; } skb_pull(skb, 3); skb_trim(skb, len); return 1; } static struct sk_buff *dm9601_tx_fixup(struct usbnet *dev, struct sk_buff *skb, gfp_t flags) { int len, pad; /* format: b1: packet length low b2: packet length high b3..n: packet data */ len = skb->len + DM_TX_OVERHEAD; /* workaround for dm962x errata with tx fifo getting out of * sync if a USB bulk transfer retry happens right after a * packet with odd / maxpacket length by adding up to 3 bytes * padding. */ while ((len & 1) || !(len % dev->maxpacket)) len++; len -= DM_TX_OVERHEAD; /* hw header doesn't count as part of length */ pad = len - skb->len; if (skb_headroom(skb) < DM_TX_OVERHEAD || skb_tailroom(skb) < pad) { struct sk_buff *skb2; skb2 = skb_copy_expand(skb, DM_TX_OVERHEAD, pad, flags); dev_kfree_skb_any(skb); skb = skb2; if (!skb) return NULL; } __skb_push(skb, DM_TX_OVERHEAD); if (pad) { memset(skb->data + skb->len, 0, pad); __skb_put(skb, pad); } skb->data[0] = len; skb->data[1] = len >> 8; return skb; } static void dm9601_status(struct usbnet *dev, struct urb *urb) { int link; u8 *buf; /* format: b0: net status b1: tx status 1 b2: tx status 2 b3: rx status b4: rx overflow b5: rx count b6: tx count b7: gpr */ if (urb->actual_length < 8) return; buf = urb->transfer_buffer; link = !!(buf[0] & 0x40); if (netif_carrier_ok(dev->net) != link) { usbnet_link_change(dev, link, 1); netdev_dbg(dev->net, "Link Status is: %d\n", link); } } static int dm9601_link_reset(struct usbnet *dev) { struct ethtool_cmd ecmd = { .cmd = ETHTOOL_GSET }; mii_check_media(&dev->mii, 1, 1); mii_ethtool_gset(&dev->mii, &ecmd); netdev_dbg(dev->net, "link_reset() speed: %u duplex: %d\n", ethtool_cmd_speed(&ecmd), ecmd.duplex); return 0; } static const struct driver_info dm9601_info = { .description = "Davicom DM96xx USB 10/100 Ethernet", .flags = FLAG_ETHER | FLAG_LINK_INTR, .bind = dm9601_bind, .rx_fixup = dm9601_rx_fixup, .tx_fixup = dm9601_tx_fixup, .status = dm9601_status, .link_reset = dm9601_link_reset, .reset = dm9601_link_reset, }; static const struct usb_device_id products[] = { { USB_DEVICE(0x07aa, 0x9601), /* Corega FEther USB-TXC */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0a46, 0x9601), /* Davicom USB-100 */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0a46, 0x6688), /* ZT6688 USB NIC */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0a46, 0x0268), /* ShanTou ST268 USB NIC */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0a46, 0x8515), /* ADMtek ADM8515 USB NIC */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0a47, 0x9601), /* Hirose USB-100 */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0fe6, 0x8101), /* DM9601 USB to Fast Ethernet Adapter */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0fe6, 0x9700), /* DM9601 USB to Fast Ethernet Adapter */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0a46, 0x9000), /* DM9000E */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0a46, 0x9620), /* DM9620 USB to Fast Ethernet Adapter */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0a46, 0x9621), /* DM9621A USB to Fast Ethernet Adapter */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0a46, 0x9622), /* DM9622 USB to Fast Ethernet Adapter */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0a46, 0x0269), /* DM962OA USB to Fast Ethernet Adapter */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0a46, 0x1269), /* DM9621A USB to Fast Ethernet Adapter */ .driver_info = (unsigned long)&dm9601_info, }, { USB_DEVICE(0x0586, 0x3427), /* ZyXEL Keenetic Plus DSL xDSL modem */ .driver_info = (unsigned long)&dm9601_info, }, {}, // END }; MODULE_DEVICE_TABLE(usb, products); static struct usb_driver dm9601_driver = { .name = "dm9601", .id_table = products, .probe = usbnet_probe, .disconnect = usbnet_disconnect, .suspend = usbnet_suspend, .resume = usbnet_resume, .disable_hub_initiated_lpm = 1, }; module_usb_driver(dm9601_driver); MODULE_AUTHOR("Peter Korsgaard <jacmet@sunsite.dk>"); MODULE_DESCRIPTION("Davicom DM96xx USB 10/100 ethernet devices"); MODULE_LICENSE("GPL"); |
| 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 10 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 | // SPDX-License-Identifier: GPL-2.0 #include "bcachefs.h" #include "bkey_buf.h" #include "bset.h" #include "btree_cache.h" #include "btree_journal_iter.h" #include "journal_io.h" #include <linux/sort.h> /* * For managing keys we read from the journal: until journal replay works normal * btree lookups need to be able to find and return keys from the journal where * they overwrite what's in the btree, so we have a special iterator and * operations for the regular btree iter code to use: */ static inline size_t idx_to_pos(struct journal_keys *keys, size_t idx) { size_t gap_size = keys->size - keys->nr; if (idx >= keys->gap) idx += gap_size; return idx; } static inline struct journal_key *idx_to_key(struct journal_keys *keys, size_t idx) { return keys->data + idx_to_pos(keys, idx); } static size_t __bch2_journal_key_search(struct journal_keys *keys, enum btree_id id, unsigned level, struct bpos pos) { size_t l = 0, r = keys->nr, m; while (l < r) { m = l + ((r - l) >> 1); if (__journal_key_cmp(id, level, pos, idx_to_key(keys, m)) > 0) l = m + 1; else r = m; } BUG_ON(l < keys->nr && __journal_key_cmp(id, level, pos, idx_to_key(keys, l)) > 0); BUG_ON(l && __journal_key_cmp(id, level, pos, idx_to_key(keys, l - 1)) <= 0); return l; } static size_t bch2_journal_key_search(struct journal_keys *keys, enum btree_id id, unsigned level, struct bpos pos) { return idx_to_pos(keys, __bch2_journal_key_search(keys, id, level, pos)); } /* Returns first non-overwritten key >= search key: */ struct bkey_i *bch2_journal_keys_peek_upto(struct bch_fs *c, enum btree_id btree_id, unsigned level, struct bpos pos, struct bpos end_pos, size_t *idx) { struct journal_keys *keys = &c->journal_keys; unsigned iters = 0; struct journal_key *k; BUG_ON(*idx > keys->nr); search: if (!*idx) *idx = __bch2_journal_key_search(keys, btree_id, level, pos); while (*idx && __journal_key_cmp(btree_id, level, end_pos, idx_to_key(keys, *idx - 1)) <= 0) { --(*idx); iters++; if (iters == 10) { *idx = 0; goto search; } } while ((k = *idx < keys->nr ? idx_to_key(keys, *idx) : NULL)) { if (__journal_key_cmp(btree_id, level, end_pos, k) < 0) return NULL; if (k->overwritten) { (*idx)++; continue; } if (__journal_key_cmp(btree_id, level, pos, k) <= 0) return k->k; (*idx)++; iters++; if (iters == 10) { *idx = 0; goto search; } } return NULL; } struct bkey_i *bch2_journal_keys_peek_slot(struct bch_fs *c, enum btree_id btree_id, unsigned level, struct bpos pos) { size_t idx = 0; return bch2_journal_keys_peek_upto(c, btree_id, level, pos, pos, &idx); } static void journal_iter_verify(struct journal_iter *iter) { struct journal_keys *keys = iter->keys; size_t gap_size = keys->size - keys->nr; BUG_ON(iter->idx >= keys->gap && iter->idx < keys->gap + gap_size); if (iter->idx < keys->size) { struct journal_key *k = keys->data + iter->idx; int cmp = cmp_int(k->btree_id, iter->btree_id) ?: cmp_int(k->level, iter->level); BUG_ON(cmp < 0); } } static void journal_iters_fix(struct bch_fs *c) { struct journal_keys *keys = &c->journal_keys; /* The key we just inserted is immediately before the gap: */ size_t gap_end = keys->gap + (keys->size - keys->nr); struct journal_key *new_key = &keys->data[keys->gap - 1]; struct journal_iter *iter; /* * If an iterator points one after the key we just inserted, decrement * the iterator so it points at the key we just inserted - if the * decrement was unnecessary, bch2_btree_and_journal_iter_peek() will * handle that: */ list_for_each_entry(iter, &c->journal_iters, list) { journal_iter_verify(iter); if (iter->idx == gap_end && new_key->btree_id == iter->btree_id && new_key->level == iter->level) iter->idx = keys->gap - 1; journal_iter_verify(iter); } } static void journal_iters_move_gap(struct bch_fs *c, size_t old_gap, size_t new_gap) { struct journal_keys *keys = &c->journal_keys; struct journal_iter *iter; size_t gap_size = keys->size - keys->nr; list_for_each_entry(iter, &c->journal_iters, list) { if (iter->idx > old_gap) iter->idx -= gap_size; if (iter->idx >= new_gap) iter->idx += gap_size; } } int bch2_journal_key_insert_take(struct bch_fs *c, enum btree_id id, unsigned level, struct bkey_i *k) { struct journal_key n = { .btree_id = id, .level = level, .k = k, .allocated = true, /* * Ensure these keys are done last by journal replay, to unblock * journal reclaim: */ .journal_seq = U32_MAX, }; struct journal_keys *keys = &c->journal_keys; size_t idx = bch2_journal_key_search(keys, id, level, k->k.p); BUG_ON(test_bit(BCH_FS_rw, &c->flags)); if (idx < keys->size && journal_key_cmp(&n, &keys->data[idx]) == 0) { if (keys->data[idx].allocated) kfree(keys->data[idx].k); keys->data[idx] = n; return 0; } if (idx > keys->gap) idx -= keys->size - keys->nr; size_t old_gap = keys->gap; if (keys->nr == keys->size) { journal_iters_move_gap(c, old_gap, keys->size); old_gap = keys->size; struct journal_keys new_keys = { .nr = keys->nr, .size = max_t(size_t, keys->size, 8) * 2, }; new_keys.data = kvmalloc_array(new_keys.size, sizeof(new_keys.data[0]), GFP_KERNEL); if (!new_keys.data) { bch_err(c, "%s: error allocating new key array (size %zu)", __func__, new_keys.size); return -BCH_ERR_ENOMEM_journal_key_insert; } /* Since @keys was full, there was no gap: */ memcpy(new_keys.data, keys->data, sizeof(keys->data[0]) * keys->nr); kvfree(keys->data); keys->data = new_keys.data; keys->nr = new_keys.nr; keys->size = new_keys.size; /* And now the gap is at the end: */ keys->gap = keys->nr; } journal_iters_move_gap(c, old_gap, idx); move_gap(keys, idx); keys->nr++; keys->data[keys->gap++] = n; journal_iters_fix(c); return 0; } /* * Can only be used from the recovery thread while we're still RO - can't be * used once we've got RW, as journal_keys is at that point used by multiple * threads: */ int bch2_journal_key_insert(struct bch_fs *c, enum btree_id id, unsigned level, struct bkey_i *k) { struct bkey_i *n; int ret; n = kmalloc(bkey_bytes(&k->k), GFP_KERNEL); if (!n) return -BCH_ERR_ENOMEM_journal_key_insert; bkey_copy(n, k); ret = bch2_journal_key_insert_take(c, id, level, n); if (ret) kfree(n); return ret; } int bch2_journal_key_delete(struct bch_fs *c, enum btree_id id, unsigned level, struct bpos pos) { struct bkey_i whiteout; bkey_init(&whiteout.k); whiteout.k.p = pos; return bch2_journal_key_insert(c, id, level, &whiteout); } bool bch2_key_deleted_in_journal(struct btree_trans *trans, enum btree_id btree, unsigned level, struct bpos pos) { struct journal_keys *keys = &trans->c->journal_keys; size_t idx = bch2_journal_key_search(keys, btree, level, pos); if (!trans->journal_replay_not_finished) return false; return (idx < keys->size && keys->data[idx].btree_id == btree && keys->data[idx].level == level && bpos_eq(keys->data[idx].k->k.p, pos) && bkey_deleted(&keys->data[idx].k->k)); } void bch2_journal_key_overwritten(struct bch_fs *c, enum btree_id btree, unsigned level, struct bpos pos) { struct journal_keys *keys = &c->journal_keys; size_t idx = bch2_journal_key_search(keys, btree, level, pos); if (idx < keys->size && keys->data[idx].btree_id == btree && keys->data[idx].level == level && bpos_eq(keys->data[idx].k->k.p, pos)) keys->data[idx].overwritten = true; } static void bch2_journal_iter_advance(struct journal_iter *iter) { if (iter->idx < iter->keys->size) { iter->idx++; if (iter->idx == iter->keys->gap) iter->idx += iter->keys->size - iter->keys->nr; } } static struct bkey_s_c bch2_journal_iter_peek(struct journal_iter *iter) { journal_iter_verify(iter); while (iter->idx < iter->keys->size) { struct journal_key *k = iter->keys->data + iter->idx; int cmp = cmp_int(k->btree_id, iter->btree_id) ?: cmp_int(k->level, iter->level); if (cmp > 0) break; BUG_ON(cmp); if (!k->overwritten) return bkey_i_to_s_c(k->k); bch2_journal_iter_advance(iter); } return bkey_s_c_null; } static void bch2_journal_iter_exit(struct journal_iter *iter) { list_del(&iter->list); } static void bch2_journal_iter_init(struct bch_fs *c, struct journal_iter *iter, enum btree_id id, unsigned level, struct bpos pos) { iter->btree_id = id; iter->level = level; iter->keys = &c->journal_keys; iter->idx = bch2_journal_key_search(&c->journal_keys, id, level, pos); journal_iter_verify(iter); } static struct bkey_s_c bch2_journal_iter_peek_btree(struct btree_and_journal_iter *iter) { return bch2_btree_node_iter_peek_unpack(&iter->node_iter, iter->b, &iter->unpacked); } static void bch2_journal_iter_advance_btree(struct btree_and_journal_iter *iter) { bch2_btree_node_iter_advance(&iter->node_iter, iter->b); } void bch2_btree_and_journal_iter_advance(struct btree_and_journal_iter *iter) { if (bpos_eq(iter->pos, SPOS_MAX)) iter->at_end = true; else iter->pos = bpos_successor(iter->pos); } static void btree_and_journal_iter_prefetch(struct btree_and_journal_iter *_iter) { struct btree_and_journal_iter iter = *_iter; struct bch_fs *c = iter.trans->c; unsigned level = iter.journal.level; struct bkey_buf tmp; unsigned nr = test_bit(BCH_FS_started, &c->flags) ? (level > 1 ? 0 : 2) : (level > 1 ? 1 : 16); iter.prefetch = false; bch2_bkey_buf_init(&tmp); while (nr--) { bch2_btree_and_journal_iter_advance(&iter); struct bkey_s_c k = bch2_btree_and_journal_iter_peek(&iter); if (!k.k) break; bch2_bkey_buf_reassemble(&tmp, c, k); bch2_btree_node_prefetch(iter.trans, NULL, tmp.k, iter.journal.btree_id, level - 1); } bch2_bkey_buf_exit(&tmp, c); } struct bkey_s_c bch2_btree_and_journal_iter_peek(struct btree_and_journal_iter *iter) { struct bkey_s_c btree_k, journal_k = bkey_s_c_null, ret; if (iter->prefetch && iter->journal.level) btree_and_journal_iter_prefetch(iter); again: if (iter->at_end) return bkey_s_c_null; while ((btree_k = bch2_journal_iter_peek_btree(iter)).k && bpos_lt(btree_k.k->p, iter->pos)) bch2_journal_iter_advance_btree(iter); if (iter->trans->journal_replay_not_finished) while ((journal_k = bch2_journal_iter_peek(&iter->journal)).k && bpos_lt(journal_k.k->p, iter->pos)) bch2_journal_iter_advance(&iter->journal); ret = journal_k.k && (!btree_k.k || bpos_le(journal_k.k->p, btree_k.k->p)) ? journal_k : btree_k; if (ret.k && iter->b && bpos_gt(ret.k->p, iter->b->data->max_key)) ret = bkey_s_c_null; if (ret.k) { iter->pos = ret.k->p; if (bkey_deleted(ret.k)) { bch2_btree_and_journal_iter_advance(iter); goto again; } } else { iter->pos = SPOS_MAX; iter->at_end = true; } return ret; } void bch2_btree_and_journal_iter_exit(struct btree_and_journal_iter *iter) { bch2_journal_iter_exit(&iter->journal); } void __bch2_btree_and_journal_iter_init_node_iter(struct btree_trans *trans, struct btree_and_journal_iter *iter, struct btree *b, struct btree_node_iter node_iter, struct bpos pos) { memset(iter, 0, sizeof(*iter)); iter->trans = trans; iter->b = b; iter->node_iter = node_iter; iter->pos = b->data->min_key; iter->at_end = false; INIT_LIST_HEAD(&iter->journal.list); if (trans->journal_replay_not_finished) { bch2_journal_iter_init(trans->c, &iter->journal, b->c.btree_id, b->c.level, pos); if (!test_bit(BCH_FS_may_go_rw, &trans->c->flags)) list_add(&iter->journal.list, &trans->c->journal_iters); } } /* * this version is used by btree_gc before filesystem has gone RW and * multithreaded, so uses the journal_iters list: */ void bch2_btree_and_journal_iter_init_node_iter(struct btree_trans *trans, struct btree_and_journal_iter *iter, struct btree *b) { struct btree_node_iter node_iter; bch2_btree_node_iter_init_from_start(&node_iter, b); __bch2_btree_and_journal_iter_init_node_iter(trans, iter, b, node_iter, b->data->min_key); } /* sort and dedup all keys in the journal: */ void bch2_journal_entries_free(struct bch_fs *c) { struct journal_replay **i; struct genradix_iter iter; genradix_for_each(&c->journal_entries, iter, i) kvfree(*i); genradix_free(&c->journal_entries); } /* * When keys compare equal, oldest compares first: */ static int journal_sort_key_cmp(const void *_l, const void *_r) { const struct journal_key *l = _l; const struct journal_key *r = _r; return journal_key_cmp(l, r) ?: cmp_int(l->journal_seq, r->journal_seq) ?: cmp_int(l->journal_offset, r->journal_offset); } void bch2_journal_keys_put(struct bch_fs *c) { struct journal_keys *keys = &c->journal_keys; BUG_ON(atomic_read(&keys->ref) <= 0); if (!atomic_dec_and_test(&keys->ref)) return; move_gap(keys, keys->nr); darray_for_each(*keys, i) if (i->allocated) kfree(i->k); kvfree(keys->data); keys->data = NULL; keys->nr = keys->gap = keys->size = 0; bch2_journal_entries_free(c); } static void __journal_keys_sort(struct journal_keys *keys) { sort(keys->data, keys->nr, sizeof(keys->data[0]), journal_sort_key_cmp, NULL); cond_resched(); struct journal_key *dst = keys->data; darray_for_each(*keys, src) { /* * We don't accumulate accounting keys here because we have to * compare each individual accounting key against the version in * the btree during replay: */ if (src->k->k.type != KEY_TYPE_accounting && src + 1 < &darray_top(*keys) && !journal_key_cmp(src, src + 1)) continue; *dst++ = *src; } keys->nr = dst - keys->data; } int bch2_journal_keys_sort(struct bch_fs *c) { struct genradix_iter iter; struct journal_replay *i, **_i; struct journal_keys *keys = &c->journal_keys; size_t nr_read = 0; genradix_for_each(&c->journal_entries, iter, _i) { i = *_i; if (journal_replay_ignore(i)) continue; cond_resched(); for_each_jset_key(k, entry, &i->j) { struct journal_key n = (struct journal_key) { .btree_id = entry->btree_id, .level = entry->level, .k = k, .journal_seq = le64_to_cpu(i->j.seq), .journal_offset = k->_data - i->j._data, }; if (darray_push(keys, n)) { __journal_keys_sort(keys); if (keys->nr * 8 > keys->size * 7) { bch_err(c, "Too many journal keys for slowpath; have %zu compacted, buf size %zu, processed %zu keys at seq %llu", keys->nr, keys->size, nr_read, le64_to_cpu(i->j.seq)); return -BCH_ERR_ENOMEM_journal_keys_sort; } BUG_ON(darray_push(keys, n)); } nr_read++; } } __journal_keys_sort(keys); keys->gap = keys->nr; bch_verbose(c, "Journal keys: %zu read, %zu after sorting and compacting", nr_read, keys->nr); return 0; } void bch2_shoot_down_journal_keys(struct bch_fs *c, enum btree_id btree, unsigned level_min, unsigned level_max, struct bpos start, struct bpos end) { struct journal_keys *keys = &c->journal_keys; size_t dst = 0; move_gap(keys, keys->nr); darray_for_each(*keys, i) if (!(i->btree_id == btree && i->level >= level_min && i->level <= level_max && bpos_ge(i->k->k.p, start) && bpos_le(i->k->k.p, end))) keys->data[dst++] = *i; keys->nr = keys->gap = dst; } void bch2_journal_keys_dump(struct bch_fs *c) { struct journal_keys *keys = &c->journal_keys; struct printbuf buf = PRINTBUF; pr_info("%zu keys:", keys->nr); move_gap(keys, keys->nr); darray_for_each(*keys, i) { printbuf_reset(&buf); bch2_bkey_val_to_text(&buf, c, bkey_i_to_s_c(i->k)); pr_err("%s l=%u %s", bch2_btree_id_str(i->btree_id), i->level, buf.buf); } printbuf_exit(&buf); } |
| 24 24 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687 4688 4689 4690 4691 4692 4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947 4948 4949 4950 4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103 5104 5105 5106 5107 5108 5109 5110 5111 5112 5113 5114 5115 5116 5117 5118 5119 5120 5121 5122 5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197 5198 5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215 5216 5217 5218 5219 5220 5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232 5233 5234 5235 5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250 5251 5252 5253 5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312 5313 5314 5315 5316 5317 5318 5319 5320 5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454 5455 5456 5457 5458 5459 5460 5461 5462 5463 5464 5465 5466 5467 5468 5469 5470 5471 5472 5473 5474 5475 5476 5477 5478 5479 5480 5481 5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575 5576 5577 5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638 5639 5640 5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806 5807 5808 5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823 5824 5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835 5836 5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127 6128 6129 6130 6131 6132 6133 6134 6135 6136 6137 6138 6139 6140 6141 6142 6143 6144 6145 6146 6147 6148 6149 6150 6151 6152 6153 6154 6155 6156 6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173 6174 6175 6176 6177 6178 6179 6180 6181 6182 6183 6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276 6277 6278 6279 6280 6281 6282 6283 6284 6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322 6323 6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356 6357 6358 6359 6360 6361 6362 6363 6364 6365 6366 6367 6368 6369 6370 6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424 6425 6426 6427 6428 6429 6430 6431 6432 6433 6434 6435 6436 6437 6438 6439 6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483 6484 6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496 6497 6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510 6511 6512 6513 6514 6515 6516 6517 6518 6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575 6576 6577 6578 6579 6580 6581 6582 6583 6584 6585 6586 6587 6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614 6615 6616 6617 6618 6619 6620 6621 6622 6623 6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634 6635 6636 6637 6638 6639 6640 6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665 6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677 6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742 6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759 6760 6761 6762 6763 6764 6765 6766 6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777 6778 6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844 6845 6846 6847 6848 6849 6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003 7004 7005 7006 7007 7008 7009 7010 7011 7012 7013 7014 7015 7016 7017 7018 7019 7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069 7070 7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096 7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120 7121 7122 7123 7124 7125 7126 7127 7128 7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139 7140 7141 7142 7143 7144 7145 7146 7147 7148 7149 7150 7151 7152 7153 7154 7155 7156 7157 7158 7159 7160 7161 7162 7163 7164 7165 7166 7167 7168 7169 7170 7171 7172 7173 7174 7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204 7205 7206 7207 7208 7209 7210 7211 7212 7213 7214 7215 7216 7217 7218 7219 7220 7221 7222 7223 7224 7225 7226 7227 7228 7229 7230 7231 7232 7233 7234 7235 7236 7237 7238 7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250 7251 7252 7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266 7267 7268 7269 7270 7271 7272 7273 7274 7275 7276 7277 7278 7279 7280 7281 7282 7283 7284 7285 7286 7287 7288 7289 7290 7291 7292 7293 7294 7295 7296 7297 7298 7299 7300 7301 7302 7303 7304 7305 7306 7307 7308 7309 7310 7311 7312 7313 7314 7315 7316 7317 7318 7319 7320 7321 7322 7323 7324 7325 7326 7327 7328 7329 7330 7331 7332 7333 7334 7335 7336 7337 7338 7339 7340 7341 7342 7343 7344 7345 7346 7347 7348 7349 7350 7351 7352 7353 7354 7355 7356 7357 7358 7359 7360 7361 7362 7363 7364 7365 7366 7367 7368 7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383 7384 7385 7386 7387 7388 7389 7390 7391 7392 7393 7394 7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438 7439 7440 7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474 7475 7476 7477 7478 7479 7480 7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494 7495 7496 7497 7498 7499 7500 7501 7502 7503 7504 7505 7506 7507 7508 7509 7510 7511 7512 7513 7514 7515 7516 7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554 7555 7556 7557 7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628 7629 7630 7631 7632 7633 7634 7635 7636 7637 7638 7639 7640 7641 7642 7643 7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673 7674 7675 7676 7677 7678 7679 7680 7681 7682 7683 7684 7685 7686 7687 7688 7689 7690 7691 7692 7693 7694 7695 7696 7697 7698 7699 7700 7701 7702 7703 7704 7705 7706 7707 7708 7709 7710 7711 7712 7713 7714 7715 7716 7717 7718 7719 7720 7721 7722 7723 7724 7725 7726 7727 7728 7729 7730 7731 7732 7733 7734 7735 7736 7737 7738 7739 7740 7741 7742 7743 7744 7745 7746 7747 7748 7749 7750 7751 7752 7753 7754 7755 7756 7757 7758 7759 7760 7761 7762 7763 7764 7765 7766 7767 7768 7769 7770 7771 7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792 7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818 7819 7820 7821 7822 7823 7824 7825 7826 7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841 7842 7843 7844 7845 7846 7847 7848 7849 7850 7851 7852 7853 7854 7855 7856 7857 7858 7859 7860 7861 7862 7863 7864 7865 7866 7867 7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885 7886 7887 7888 7889 7890 7891 7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944 7945 7946 7947 7948 7949 7950 7951 7952 7953 7954 7955 7956 7957 7958 7959 7960 7961 7962 7963 7964 7965 7966 7967 7968 7969 7970 7971 7972 7973 7974 7975 7976 7977 7978 7979 7980 7981 7982 7983 7984 7985 7986 7987 7988 7989 7990 7991 7992 7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004 8005 8006 8007 8008 8009 8010 8011 8012 8013 8014 8015 8016 8017 8018 8019 8020 8021 8022 8023 8024 8025 8026 8027 8028 8029 8030 8031 8032 8033 8034 8035 8036 8037 8038 8039 8040 8041 8042 8043 8044 8045 8046 8047 8048 8049 8050 8051 8052 8053 8054 8055 8056 8057 8058 8059 8060 8061 8062 8063 8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076 8077 8078 8079 8080 8081 8082 8083 8084 8085 8086 8087 8088 8089 8090 8091 8092 8093 8094 8095 8096 8097 8098 8099 8100 8101 8102 8103 8104 8105 8106 8107 8108 8109 8110 8111 8112 8113 8114 8115 8116 8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127 8128 8129 8130 8131 8132 8133 8134 8135 8136 8137 8138 8139 8140 8141 8142 8143 8144 8145 8146 8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160 8161 8162 8163 8164 8165 8166 8167 8168 8169 8170 8171 8172 8173 8174 8175 8176 8177 8178 8179 8180 8181 8182 8183 8184 8185 8186 8187 8188 8189 8190 8191 8192 8193 8194 8195 8196 8197 8198 8199 8200 8201 8202 8203 8204 8205 8206 8207 8208 8209 8210 8211 8212 8213 8214 8215 8216 8217 8218 8219 8220 8221 8222 8223 8224 8225 8226 8227 8228 8229 8230 8231 8232 8233 8234 8235 8236 8237 8238 8239 8240 8241 8242 8243 8244 8245 8246 8247 8248 8249 8250 8251 8252 8253 8254 8255 8256 8257 8258 8259 8260 8261 8262 8263 8264 8265 8266 8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308 8309 8310 8311 8312 8313 8314 8315 8316 8317 8318 8319 8320 8321 8322 8323 8324 8325 8326 8327 8328 8329 8330 8331 8332 8333 8334 8335 8336 8337 8338 8339 8340 8341 8342 8343 8344 8345 8346 8347 8348 8349 8350 8351 8352 8353 8354 8355 8356 8357 8358 8359 8360 8361 8362 8363 8364 8365 8366 8367 8368 8369 8370 8371 8372 8373 8374 8375 8376 8377 8378 8379 8380 8381 8382 8383 8384 8385 8386 8387 8388 8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399 8400 8401 8402 8403 8404 8405 8406 8407 8408 8409 8410 8411 8412 8413 8414 8415 8416 8417 8418 8419 8420 8421 8422 8423 8424 8425 8426 8427 8428 8429 8430 8431 8432 8433 8434 8435 8436 8437 8438 8439 8440 8441 8442 8443 8444 8445 8446 8447 8448 8449 8450 8451 8452 8453 8454 8455 8456 8457 8458 8459 8460 8461 8462 8463 8464 8465 8466 8467 8468 8469 8470 8471 8472 8473 8474 8475 8476 8477 8478 8479 8480 8481 8482 8483 8484 8485 8486 8487 8488 8489 8490 8491 8492 8493 8494 8495 8496 8497 8498 8499 8500 8501 8502 8503 8504 8505 8506 8507 8508 8509 8510 8511 8512 8513 8514 8515 8516 8517 8518 8519 8520 8521 8522 8523 8524 8525 8526 8527 8528 8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541 8542 8543 8544 8545 8546 8547 8548 8549 8550 8551 8552 8553 8554 8555 8556 8557 8558 8559 8560 8561 8562 8563 8564 8565 8566 8567 8568 8569 8570 8571 8572 8573 8574 8575 8576 8577 8578 8579 8580 8581 8582 8583 8584 8585 8586 8587 8588 8589 8590 8591 8592 8593 8594 8595 8596 8597 8598 8599 8600 8601 8602 8603 8604 8605 8606 8607 8608 8609 8610 8611 8612 8613 8614 8615 8616 8617 8618 8619 8620 8621 8622 8623 8624 8625 8626 8627 8628 8629 8630 8631 8632 8633 8634 8635 8636 8637 8638 8639 8640 8641 8642 8643 8644 8645 8646 8647 8648 8649 8650 8651 8652 8653 8654 8655 8656 8657 8658 8659 8660 8661 8662 8663 8664 8665 8666 8667 8668 8669 8670 8671 8672 8673 8674 8675 8676 8677 8678 8679 8680 8681 8682 8683 8684 8685 8686 8687 8688 8689 8690 8691 8692 8693 8694 8695 8696 8697 8698 8699 8700 8701 8702 8703 8704 8705 8706 8707 8708 8709 8710 8711 8712 8713 8714 8715 8716 8717 8718 8719 8720 8721 8722 8723 8724 8725 8726 8727 8728 8729 8730 8731 8732 8733 8734 8735 8736 8737 8738 8739 8740 8741 8742 8743 8744 8745 8746 8747 8748 8749 8750 8751 8752 8753 8754 8755 8756 8757 8758 8759 8760 8761 8762 8763 8764 8765 8766 8767 8768 8769 8770 8771 8772 8773 8774 8775 8776 8777 8778 8779 8780 8781 8782 8783 8784 8785 8786 8787 8788 8789 8790 8791 8792 8793 8794 8795 8796 8797 8798 8799 8800 8801 8802 8803 8804 8805 8806 8807 8808 8809 8810 8811 8812 8813 8814 8815 8816 8817 8818 8819 8820 8821 8822 8823 8824 8825 8826 8827 8828 8829 8830 8831 8832 8833 8834 8835 8836 8837 8838 8839 8840 8841 8842 8843 8844 8845 8846 8847 8848 8849 8850 8851 8852 8853 8854 8855 8856 8857 8858 8859 8860 8861 8862 8863 8864 8865 8866 8867 8868 8869 8870 8871 8872 8873 8874 8875 8876 8877 8878 8879 8880 8881 8882 8883 8884 8885 8886 8887 8888 8889 8890 8891 8892 8893 8894 8895 8896 8897 8898 8899 8900 8901 8902 8903 8904 8905 8906 8907 8908 8909 8910 8911 8912 8913 8914 8915 8916 8917 8918 8919 8920 8921 8922 8923 8924 8925 8926 8927 8928 8929 8930 8931 8932 8933 8934 8935 8936 8937 8938 8939 8940 8941 8942 8943 8944 8945 8946 8947 8948 8949 8950 8951 8952 8953 8954 8955 8956 8957 8958 8959 8960 8961 8962 8963 8964 8965 8966 8967 8968 8969 8970 8971 8972 8973 8974 8975 8976 8977 8978 8979 8980 8981 8982 8983 8984 8985 8986 8987 8988 8989 8990 8991 8992 8993 8994 8995 8996 8997 8998 8999 9000 9001 9002 9003 9004 9005 9006 9007 9008 9009 9010 9011 9012 9013 9014 9015 9016 9017 9018 9019 9020 9021 9022 9023 9024 9025 9026 9027 9028 9029 9030 9031 9032 9033 9034 9035 9036 9037 9038 9039 9040 9041 9042 9043 9044 9045 9046 9047 9048 9049 9050 9051 9052 9053 9054 9055 9056 9057 9058 9059 9060 9061 9062 9063 9064 9065 9066 9067 9068 9069 9070 9071 9072 9073 9074 9075 9076 9077 9078 9079 9080 9081 9082 9083 9084 9085 9086 9087 9088 9089 9090 9091 9092 9093 9094 9095 9096 9097 9098 9099 9100 9101 9102 9103 9104 9105 9106 9107 9108 9109 9110 9111 9112 9113 9114 9115 9116 9117 9118 9119 9120 9121 9122 9123 9124 9125 9126 9127 9128 9129 9130 9131 9132 9133 9134 9135 9136 9137 9138 9139 9140 9141 9142 9143 9144 9145 9146 9147 9148 9149 9150 9151 9152 9153 9154 9155 9156 9157 9158 9159 9160 9161 9162 9163 9164 9165 9166 9167 9168 9169 9170 9171 9172 9173 9174 9175 9176 9177 9178 9179 9180 9181 9182 9183 9184 9185 9186 9187 9188 9189 9190 9191 9192 9193 9194 9195 9196 9197 9198 9199 9200 9201 9202 9203 9204 9205 9206 9207 9208 9209 9210 9211 9212 9213 9214 9215 9216 9217 9218 9219 9220 9221 9222 9223 9224 9225 9226 9227 9228 9229 9230 9231 9232 9233 9234 9235 9236 9237 9238 9239 9240 9241 9242 9243 9244 9245 9246 9247 9248 9249 9250 9251 9252 9253 9254 9255 9256 9257 9258 9259 9260 9261 9262 9263 9264 9265 9266 9267 9268 9269 9270 9271 9272 9273 9274 9275 9276 9277 9278 9279 9280 9281 9282 9283 9284 9285 9286 9287 9288 9289 9290 9291 9292 9293 9294 9295 9296 9297 9298 9299 9300 9301 9302 9303 9304 9305 9306 9307 9308 9309 9310 9311 9312 9313 9314 9315 9316 9317 9318 9319 9320 9321 9322 9323 9324 9325 9326 9327 9328 9329 9330 9331 9332 9333 9334 9335 9336 9337 9338 9339 9340 9341 9342 9343 9344 9345 9346 9347 9348 9349 9350 9351 9352 9353 9354 9355 9356 9357 9358 9359 9360 9361 9362 9363 9364 9365 9366 9367 9368 9369 9370 9371 9372 9373 9374 9375 9376 9377 9378 9379 9380 9381 9382 9383 9384 9385 9386 9387 9388 9389 9390 9391 9392 9393 9394 9395 9396 9397 9398 9399 9400 9401 9402 9403 9404 9405 9406 9407 9408 9409 9410 9411 9412 9413 9414 9415 9416 9417 9418 9419 9420 9421 9422 9423 9424 9425 9426 9427 9428 9429 9430 9431 9432 9433 9434 9435 9436 9437 9438 9439 9440 9441 9442 9443 9444 9445 9446 9447 9448 9449 9450 9451 9452 9453 9454 9455 9456 9457 9458 9459 9460 9461 9462 9463 9464 9465 9466 9467 9468 9469 9470 9471 9472 9473 9474 9475 9476 9477 9478 9479 9480 9481 9482 9483 9484 9485 9486 9487 9488 9489 9490 9491 9492 9493 9494 9495 9496 9497 9498 9499 9500 9501 9502 9503 9504 9505 9506 9507 9508 9509 9510 9511 9512 9513 9514 9515 9516 9517 9518 9519 9520 9521 9522 9523 9524 9525 9526 9527 9528 9529 9530 9531 9532 9533 9534 9535 9536 9537 9538 9539 9540 9541 9542 9543 9544 9545 9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556 9557 9558 9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572 9573 9574 9575 9576 9577 9578 9579 9580 9581 9582 9583 9584 9585 9586 9587 9588 9589 9590 9591 9592 9593 9594 9595 9596 9597 9598 9599 9600 9601 9602 9603 9604 9605 9606 9607 9608 9609 9610 9611 9612 9613 9614 9615 9616 9617 9618 9619 9620 9621 9622 9623 9624 9625 9626 9627 9628 9629 9630 9631 9632 9633 9634 | // SPDX-License-Identifier: ISC /* * Copyright (c) 2005-2011 Atheros Communications Inc. * Copyright (c) 2011-2017 Qualcomm Atheros, Inc. * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. * Copyright (c) 2021-2024 Qualcomm Innovation Center, Inc. All rights reserved. */ #include <linux/skbuff.h> #include <linux/ctype.h> #include "core.h" #include "htc.h" #include "debug.h" #include "wmi.h" #include "wmi-tlv.h" #include "mac.h" #include "testmode.h" #include "wmi-ops.h" #include "p2p.h" #include "hw.h" #include "hif.h" #include "txrx.h" #define ATH10K_WMI_BARRIER_ECHO_ID 0xBA991E9 #define ATH10K_WMI_BARRIER_TIMEOUT_HZ (3 * HZ) #define ATH10K_WMI_DFS_CONF_TIMEOUT_HZ (HZ / 6) /* MAIN WMI cmd track */ static struct wmi_cmd_map wmi_cmd_map = { .init_cmdid = WMI_INIT_CMDID, .start_scan_cmdid = WMI_START_SCAN_CMDID, .stop_scan_cmdid = WMI_STOP_SCAN_CMDID, .scan_chan_list_cmdid = WMI_SCAN_CHAN_LIST_CMDID, .scan_sch_prio_tbl_cmdid = WMI_SCAN_SCH_PRIO_TBL_CMDID, .scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_regdomain_cmdid = WMI_PDEV_SET_REGDOMAIN_CMDID, .pdev_set_channel_cmdid = WMI_PDEV_SET_CHANNEL_CMDID, .pdev_set_param_cmdid = WMI_PDEV_SET_PARAM_CMDID, .pdev_pktlog_enable_cmdid = WMI_PDEV_PKTLOG_ENABLE_CMDID, .pdev_pktlog_disable_cmdid = WMI_PDEV_PKTLOG_DISABLE_CMDID, .pdev_set_wmm_params_cmdid = WMI_PDEV_SET_WMM_PARAMS_CMDID, .pdev_set_ht_cap_ie_cmdid = WMI_PDEV_SET_HT_CAP_IE_CMDID, .pdev_set_vht_cap_ie_cmdid = WMI_PDEV_SET_VHT_CAP_IE_CMDID, .pdev_set_dscp_tid_map_cmdid = WMI_PDEV_SET_DSCP_TID_MAP_CMDID, .pdev_set_quiet_mode_cmdid = WMI_PDEV_SET_QUIET_MODE_CMDID, .pdev_green_ap_ps_enable_cmdid = WMI_PDEV_GREEN_AP_PS_ENABLE_CMDID, .pdev_get_tpc_config_cmdid = WMI_PDEV_GET_TPC_CONFIG_CMDID, .pdev_set_base_macaddr_cmdid = WMI_PDEV_SET_BASE_MACADDR_CMDID, .vdev_create_cmdid = WMI_VDEV_CREATE_CMDID, .vdev_delete_cmdid = WMI_VDEV_DELETE_CMDID, .vdev_start_request_cmdid = WMI_VDEV_START_REQUEST_CMDID, .vdev_restart_request_cmdid = WMI_VDEV_RESTART_REQUEST_CMDID, .vdev_up_cmdid = WMI_VDEV_UP_CMDID, .vdev_stop_cmdid = WMI_VDEV_STOP_CMDID, .vdev_down_cmdid = WMI_VDEV_DOWN_CMDID, .vdev_set_param_cmdid = WMI_VDEV_SET_PARAM_CMDID, .vdev_install_key_cmdid = WMI_VDEV_INSTALL_KEY_CMDID, .peer_create_cmdid = WMI_PEER_CREATE_CMDID, .peer_delete_cmdid = WMI_PEER_DELETE_CMDID, .peer_flush_tids_cmdid = WMI_PEER_FLUSH_TIDS_CMDID, .peer_set_param_cmdid = WMI_PEER_SET_PARAM_CMDID, .peer_assoc_cmdid = WMI_PEER_ASSOC_CMDID, .peer_add_wds_entry_cmdid = WMI_PEER_ADD_WDS_ENTRY_CMDID, .peer_remove_wds_entry_cmdid = WMI_PEER_REMOVE_WDS_ENTRY_CMDID, .peer_mcast_group_cmdid = WMI_PEER_MCAST_GROUP_CMDID, .bcn_tx_cmdid = WMI_BCN_TX_CMDID, .pdev_send_bcn_cmdid = WMI_PDEV_SEND_BCN_CMDID, .bcn_tmpl_cmdid = WMI_BCN_TMPL_CMDID, .bcn_filter_rx_cmdid = WMI_BCN_FILTER_RX_CMDID, .prb_req_filter_rx_cmdid = WMI_PRB_REQ_FILTER_RX_CMDID, .mgmt_tx_cmdid = WMI_MGMT_TX_CMDID, .prb_tmpl_cmdid = WMI_PRB_TMPL_CMDID, .addba_clear_resp_cmdid = WMI_ADDBA_CLEAR_RESP_CMDID, .addba_send_cmdid = WMI_ADDBA_SEND_CMDID, .addba_status_cmdid = WMI_ADDBA_STATUS_CMDID, .delba_send_cmdid = WMI_DELBA_SEND_CMDID, .addba_set_resp_cmdid = WMI_ADDBA_SET_RESP_CMDID, .send_singleamsdu_cmdid = WMI_SEND_SINGLEAMSDU_CMDID, .sta_powersave_mode_cmdid = WMI_STA_POWERSAVE_MODE_CMDID, .sta_powersave_param_cmdid = WMI_STA_POWERSAVE_PARAM_CMDID, .sta_mimo_ps_mode_cmdid = WMI_STA_MIMO_PS_MODE_CMDID, .pdev_dfs_enable_cmdid = WMI_PDEV_DFS_ENABLE_CMDID, .pdev_dfs_disable_cmdid = WMI_PDEV_DFS_DISABLE_CMDID, .roam_scan_mode = WMI_ROAM_SCAN_MODE, .roam_scan_rssi_threshold = WMI_ROAM_SCAN_RSSI_THRESHOLD, .roam_scan_period = WMI_ROAM_SCAN_PERIOD, .roam_scan_rssi_change_threshold = WMI_ROAM_SCAN_RSSI_CHANGE_THRESHOLD, .roam_ap_profile = WMI_ROAM_AP_PROFILE, .ofl_scan_add_ap_profile = WMI_ROAM_AP_PROFILE, .ofl_scan_remove_ap_profile = WMI_OFL_SCAN_REMOVE_AP_PROFILE, .ofl_scan_period = WMI_OFL_SCAN_PERIOD, .p2p_dev_set_device_info = WMI_P2P_DEV_SET_DEVICE_INFO, .p2p_dev_set_discoverability = WMI_P2P_DEV_SET_DISCOVERABILITY, .p2p_go_set_beacon_ie = WMI_P2P_GO_SET_BEACON_IE, .p2p_go_set_probe_resp_ie = WMI_P2P_GO_SET_PROBE_RESP_IE, .p2p_set_vendor_ie_data_cmdid = WMI_P2P_SET_VENDOR_IE_DATA_CMDID, .ap_ps_peer_param_cmdid = WMI_AP_PS_PEER_PARAM_CMDID, .ap_ps_peer_uapsd_coex_cmdid = WMI_AP_PS_PEER_UAPSD_COEX_CMDID, .peer_rate_retry_sched_cmdid = WMI_PEER_RATE_RETRY_SCHED_CMDID, .wlan_profile_trigger_cmdid = WMI_WLAN_PROFILE_TRIGGER_CMDID, .wlan_profile_set_hist_intvl_cmdid = WMI_WLAN_PROFILE_SET_HIST_INTVL_CMDID, .wlan_profile_get_profile_data_cmdid = WMI_WLAN_PROFILE_GET_PROFILE_DATA_CMDID, .wlan_profile_enable_profile_id_cmdid = WMI_WLAN_PROFILE_ENABLE_PROFILE_ID_CMDID, .wlan_profile_list_profile_id_cmdid = WMI_WLAN_PROFILE_LIST_PROFILE_ID_CMDID, .pdev_suspend_cmdid = WMI_PDEV_SUSPEND_CMDID, .pdev_resume_cmdid = WMI_PDEV_RESUME_CMDID, .add_bcn_filter_cmdid = WMI_ADD_BCN_FILTER_CMDID, .rmv_bcn_filter_cmdid = WMI_RMV_BCN_FILTER_CMDID, .wow_add_wake_pattern_cmdid = WMI_WOW_ADD_WAKE_PATTERN_CMDID, .wow_del_wake_pattern_cmdid = WMI_WOW_DEL_WAKE_PATTERN_CMDID, .wow_enable_disable_wake_event_cmdid = WMI_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID, .wow_enable_cmdid = WMI_WOW_ENABLE_CMDID, .wow_hostwakeup_from_sleep_cmdid = WMI_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID, .rtt_measreq_cmdid = WMI_RTT_MEASREQ_CMDID, .rtt_tsf_cmdid = WMI_RTT_TSF_CMDID, .vdev_spectral_scan_configure_cmdid = WMI_VDEV_SPECTRAL_SCAN_CONFIGURE_CMDID, .vdev_spectral_scan_enable_cmdid = WMI_VDEV_SPECTRAL_SCAN_ENABLE_CMDID, .request_stats_cmdid = WMI_REQUEST_STATS_CMDID, .set_arp_ns_offload_cmdid = WMI_SET_ARP_NS_OFFLOAD_CMDID, .network_list_offload_config_cmdid = WMI_NETWORK_LIST_OFFLOAD_CONFIG_CMDID, .gtk_offload_cmdid = WMI_GTK_OFFLOAD_CMDID, .csa_offload_enable_cmdid = WMI_CSA_OFFLOAD_ENABLE_CMDID, .csa_offload_chanswitch_cmdid = WMI_CSA_OFFLOAD_CHANSWITCH_CMDID, .chatter_set_mode_cmdid = WMI_CHATTER_SET_MODE_CMDID, .peer_tid_addba_cmdid = WMI_PEER_TID_ADDBA_CMDID, .peer_tid_delba_cmdid = WMI_PEER_TID_DELBA_CMDID, .sta_dtim_ps_method_cmdid = WMI_STA_DTIM_PS_METHOD_CMDID, .sta_uapsd_auto_trig_cmdid = WMI_STA_UAPSD_AUTO_TRIG_CMDID, .sta_keepalive_cmd = WMI_STA_KEEPALIVE_CMD, .echo_cmdid = WMI_ECHO_CMDID, .pdev_utf_cmdid = WMI_PDEV_UTF_CMDID, .dbglog_cfg_cmdid = WMI_DBGLOG_CFG_CMDID, .pdev_qvit_cmdid = WMI_PDEV_QVIT_CMDID, .pdev_ftm_intg_cmdid = WMI_PDEV_FTM_INTG_CMDID, .vdev_set_keepalive_cmdid = WMI_VDEV_SET_KEEPALIVE_CMDID, .vdev_get_keepalive_cmdid = WMI_VDEV_GET_KEEPALIVE_CMDID, .force_fw_hang_cmdid = WMI_FORCE_FW_HANG_CMDID, .gpio_config_cmdid = WMI_GPIO_CONFIG_CMDID, .gpio_output_cmdid = WMI_GPIO_OUTPUT_CMDID, .pdev_get_temperature_cmdid = WMI_CMD_UNSUPPORTED, .pdev_enable_adaptive_cca_cmdid = WMI_CMD_UNSUPPORTED, .scan_update_request_cmdid = WMI_CMD_UNSUPPORTED, .vdev_standby_response_cmdid = WMI_CMD_UNSUPPORTED, .vdev_resume_response_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_add_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_evict_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_restore_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_print_all_peers_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_update_wds_entry_cmdid = WMI_CMD_UNSUPPORTED, .peer_add_proxy_sta_entry_cmdid = WMI_CMD_UNSUPPORTED, .rtt_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .oem_req_cmdid = WMI_CMD_UNSUPPORTED, .nan_cmdid = WMI_CMD_UNSUPPORTED, .vdev_ratemask_cmdid = WMI_CMD_UNSUPPORTED, .qboost_cfg_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_enable_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_set_rx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_tx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_train_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_node_config_ops_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_antenna_switch_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_ctl_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_mimogain_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_chainmsk_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_fips_cmdid = WMI_CMD_UNSUPPORTED, .tt_set_conf_cmdid = WMI_CMD_UNSUPPORTED, .fwtest_cmdid = WMI_CMD_UNSUPPORTED, .vdev_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .peer_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_cck_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_ofdm_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_reserve_ast_entry_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_nfcal_power_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ast_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_dscp_tid_map_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_filter_neighbor_rx_packets_cmdid = WMI_CMD_UNSUPPORTED, .mu_cal_start_cmdid = WMI_CMD_UNSUPPORTED, .set_cca_params_cmdid = WMI_CMD_UNSUPPORTED, .pdev_bss_chan_info_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED, .radar_found_cmdid = WMI_CMD_UNSUPPORTED, }; /* 10.X WMI cmd track */ static struct wmi_cmd_map wmi_10x_cmd_map = { .init_cmdid = WMI_10X_INIT_CMDID, .start_scan_cmdid = WMI_10X_START_SCAN_CMDID, .stop_scan_cmdid = WMI_10X_STOP_SCAN_CMDID, .scan_chan_list_cmdid = WMI_10X_SCAN_CHAN_LIST_CMDID, .scan_sch_prio_tbl_cmdid = WMI_CMD_UNSUPPORTED, .scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_regdomain_cmdid = WMI_10X_PDEV_SET_REGDOMAIN_CMDID, .pdev_set_channel_cmdid = WMI_10X_PDEV_SET_CHANNEL_CMDID, .pdev_set_param_cmdid = WMI_10X_PDEV_SET_PARAM_CMDID, .pdev_pktlog_enable_cmdid = WMI_10X_PDEV_PKTLOG_ENABLE_CMDID, .pdev_pktlog_disable_cmdid = WMI_10X_PDEV_PKTLOG_DISABLE_CMDID, .pdev_set_wmm_params_cmdid = WMI_10X_PDEV_SET_WMM_PARAMS_CMDID, .pdev_set_ht_cap_ie_cmdid = WMI_10X_PDEV_SET_HT_CAP_IE_CMDID, .pdev_set_vht_cap_ie_cmdid = WMI_10X_PDEV_SET_VHT_CAP_IE_CMDID, .pdev_set_dscp_tid_map_cmdid = WMI_10X_PDEV_SET_DSCP_TID_MAP_CMDID, .pdev_set_quiet_mode_cmdid = WMI_10X_PDEV_SET_QUIET_MODE_CMDID, .pdev_green_ap_ps_enable_cmdid = WMI_10X_PDEV_GREEN_AP_PS_ENABLE_CMDID, .pdev_get_tpc_config_cmdid = WMI_10X_PDEV_GET_TPC_CONFIG_CMDID, .pdev_set_base_macaddr_cmdid = WMI_10X_PDEV_SET_BASE_MACADDR_CMDID, .vdev_create_cmdid = WMI_10X_VDEV_CREATE_CMDID, .vdev_delete_cmdid = WMI_10X_VDEV_DELETE_CMDID, .vdev_start_request_cmdid = WMI_10X_VDEV_START_REQUEST_CMDID, .vdev_restart_request_cmdid = WMI_10X_VDEV_RESTART_REQUEST_CMDID, .vdev_up_cmdid = WMI_10X_VDEV_UP_CMDID, .vdev_stop_cmdid = WMI_10X_VDEV_STOP_CMDID, .vdev_down_cmdid = WMI_10X_VDEV_DOWN_CMDID, .vdev_set_param_cmdid = WMI_10X_VDEV_SET_PARAM_CMDID, .vdev_install_key_cmdid = WMI_10X_VDEV_INSTALL_KEY_CMDID, .peer_create_cmdid = WMI_10X_PEER_CREATE_CMDID, .peer_delete_cmdid = WMI_10X_PEER_DELETE_CMDID, .peer_flush_tids_cmdid = WMI_10X_PEER_FLUSH_TIDS_CMDID, .peer_set_param_cmdid = WMI_10X_PEER_SET_PARAM_CMDID, .peer_assoc_cmdid = WMI_10X_PEER_ASSOC_CMDID, .peer_add_wds_entry_cmdid = WMI_10X_PEER_ADD_WDS_ENTRY_CMDID, .peer_remove_wds_entry_cmdid = WMI_10X_PEER_REMOVE_WDS_ENTRY_CMDID, .peer_mcast_group_cmdid = WMI_10X_PEER_MCAST_GROUP_CMDID, .bcn_tx_cmdid = WMI_10X_BCN_TX_CMDID, .pdev_send_bcn_cmdid = WMI_10X_PDEV_SEND_BCN_CMDID, .bcn_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .bcn_filter_rx_cmdid = WMI_10X_BCN_FILTER_RX_CMDID, .prb_req_filter_rx_cmdid = WMI_10X_PRB_REQ_FILTER_RX_CMDID, .mgmt_tx_cmdid = WMI_10X_MGMT_TX_CMDID, .prb_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .addba_clear_resp_cmdid = WMI_10X_ADDBA_CLEAR_RESP_CMDID, .addba_send_cmdid = WMI_10X_ADDBA_SEND_CMDID, .addba_status_cmdid = WMI_10X_ADDBA_STATUS_CMDID, .delba_send_cmdid = WMI_10X_DELBA_SEND_CMDID, .addba_set_resp_cmdid = WMI_10X_ADDBA_SET_RESP_CMDID, .send_singleamsdu_cmdid = WMI_10X_SEND_SINGLEAMSDU_CMDID, .sta_powersave_mode_cmdid = WMI_10X_STA_POWERSAVE_MODE_CMDID, .sta_powersave_param_cmdid = WMI_10X_STA_POWERSAVE_PARAM_CMDID, .sta_mimo_ps_mode_cmdid = WMI_10X_STA_MIMO_PS_MODE_CMDID, .pdev_dfs_enable_cmdid = WMI_10X_PDEV_DFS_ENABLE_CMDID, .pdev_dfs_disable_cmdid = WMI_10X_PDEV_DFS_DISABLE_CMDID, .roam_scan_mode = WMI_10X_ROAM_SCAN_MODE, .roam_scan_rssi_threshold = WMI_10X_ROAM_SCAN_RSSI_THRESHOLD, .roam_scan_period = WMI_10X_ROAM_SCAN_PERIOD, .roam_scan_rssi_change_threshold = WMI_10X_ROAM_SCAN_RSSI_CHANGE_THRESHOLD, .roam_ap_profile = WMI_10X_ROAM_AP_PROFILE, .ofl_scan_add_ap_profile = WMI_10X_OFL_SCAN_ADD_AP_PROFILE, .ofl_scan_remove_ap_profile = WMI_10X_OFL_SCAN_REMOVE_AP_PROFILE, .ofl_scan_period = WMI_10X_OFL_SCAN_PERIOD, .p2p_dev_set_device_info = WMI_10X_P2P_DEV_SET_DEVICE_INFO, .p2p_dev_set_discoverability = WMI_10X_P2P_DEV_SET_DISCOVERABILITY, .p2p_go_set_beacon_ie = WMI_10X_P2P_GO_SET_BEACON_IE, .p2p_go_set_probe_resp_ie = WMI_10X_P2P_GO_SET_PROBE_RESP_IE, .p2p_set_vendor_ie_data_cmdid = WMI_CMD_UNSUPPORTED, .ap_ps_peer_param_cmdid = WMI_10X_AP_PS_PEER_PARAM_CMDID, .ap_ps_peer_uapsd_coex_cmdid = WMI_CMD_UNSUPPORTED, .peer_rate_retry_sched_cmdid = WMI_10X_PEER_RATE_RETRY_SCHED_CMDID, .wlan_profile_trigger_cmdid = WMI_10X_WLAN_PROFILE_TRIGGER_CMDID, .wlan_profile_set_hist_intvl_cmdid = WMI_10X_WLAN_PROFILE_SET_HIST_INTVL_CMDID, .wlan_profile_get_profile_data_cmdid = WMI_10X_WLAN_PROFILE_GET_PROFILE_DATA_CMDID, .wlan_profile_enable_profile_id_cmdid = WMI_10X_WLAN_PROFILE_ENABLE_PROFILE_ID_CMDID, .wlan_profile_list_profile_id_cmdid = WMI_10X_WLAN_PROFILE_LIST_PROFILE_ID_CMDID, .pdev_suspend_cmdid = WMI_10X_PDEV_SUSPEND_CMDID, .pdev_resume_cmdid = WMI_10X_PDEV_RESUME_CMDID, .add_bcn_filter_cmdid = WMI_10X_ADD_BCN_FILTER_CMDID, .rmv_bcn_filter_cmdid = WMI_10X_RMV_BCN_FILTER_CMDID, .wow_add_wake_pattern_cmdid = WMI_10X_WOW_ADD_WAKE_PATTERN_CMDID, .wow_del_wake_pattern_cmdid = WMI_10X_WOW_DEL_WAKE_PATTERN_CMDID, .wow_enable_disable_wake_event_cmdid = WMI_10X_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID, .wow_enable_cmdid = WMI_10X_WOW_ENABLE_CMDID, .wow_hostwakeup_from_sleep_cmdid = WMI_10X_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID, .rtt_measreq_cmdid = WMI_10X_RTT_MEASREQ_CMDID, .rtt_tsf_cmdid = WMI_10X_RTT_TSF_CMDID, .vdev_spectral_scan_configure_cmdid = WMI_10X_VDEV_SPECTRAL_SCAN_CONFIGURE_CMDID, .vdev_spectral_scan_enable_cmdid = WMI_10X_VDEV_SPECTRAL_SCAN_ENABLE_CMDID, .request_stats_cmdid = WMI_10X_REQUEST_STATS_CMDID, .set_arp_ns_offload_cmdid = WMI_CMD_UNSUPPORTED, .network_list_offload_config_cmdid = WMI_CMD_UNSUPPORTED, .gtk_offload_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_enable_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_chanswitch_cmdid = WMI_CMD_UNSUPPORTED, .chatter_set_mode_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_addba_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_delba_cmdid = WMI_CMD_UNSUPPORTED, .sta_dtim_ps_method_cmdid = WMI_CMD_UNSUPPORTED, .sta_uapsd_auto_trig_cmdid = WMI_CMD_UNSUPPORTED, .sta_keepalive_cmd = WMI_CMD_UNSUPPORTED, .echo_cmdid = WMI_10X_ECHO_CMDID, .pdev_utf_cmdid = WMI_10X_PDEV_UTF_CMDID, .dbglog_cfg_cmdid = WMI_10X_DBGLOG_CFG_CMDID, .pdev_qvit_cmdid = WMI_10X_PDEV_QVIT_CMDID, .pdev_ftm_intg_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .force_fw_hang_cmdid = WMI_CMD_UNSUPPORTED, .gpio_config_cmdid = WMI_10X_GPIO_CONFIG_CMDID, .gpio_output_cmdid = WMI_10X_GPIO_OUTPUT_CMDID, .pdev_get_temperature_cmdid = WMI_CMD_UNSUPPORTED, .pdev_enable_adaptive_cca_cmdid = WMI_CMD_UNSUPPORTED, .scan_update_request_cmdid = WMI_CMD_UNSUPPORTED, .vdev_standby_response_cmdid = WMI_CMD_UNSUPPORTED, .vdev_resume_response_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_add_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_evict_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_restore_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_print_all_peers_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_update_wds_entry_cmdid = WMI_CMD_UNSUPPORTED, .peer_add_proxy_sta_entry_cmdid = WMI_CMD_UNSUPPORTED, .rtt_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .oem_req_cmdid = WMI_CMD_UNSUPPORTED, .nan_cmdid = WMI_CMD_UNSUPPORTED, .vdev_ratemask_cmdid = WMI_CMD_UNSUPPORTED, .qboost_cfg_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_enable_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_set_rx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_tx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_train_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_node_config_ops_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_antenna_switch_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_ctl_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_mimogain_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_chainmsk_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_fips_cmdid = WMI_CMD_UNSUPPORTED, .tt_set_conf_cmdid = WMI_CMD_UNSUPPORTED, .fwtest_cmdid = WMI_CMD_UNSUPPORTED, .vdev_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .peer_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_cck_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_ofdm_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_reserve_ast_entry_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_nfcal_power_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ast_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_dscp_tid_map_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_filter_neighbor_rx_packets_cmdid = WMI_CMD_UNSUPPORTED, .mu_cal_start_cmdid = WMI_CMD_UNSUPPORTED, .set_cca_params_cmdid = WMI_CMD_UNSUPPORTED, .pdev_bss_chan_info_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED, .radar_found_cmdid = WMI_CMD_UNSUPPORTED, }; /* 10.2.4 WMI cmd track */ static struct wmi_cmd_map wmi_10_2_4_cmd_map = { .init_cmdid = WMI_10_2_INIT_CMDID, .start_scan_cmdid = WMI_10_2_START_SCAN_CMDID, .stop_scan_cmdid = WMI_10_2_STOP_SCAN_CMDID, .scan_chan_list_cmdid = WMI_10_2_SCAN_CHAN_LIST_CMDID, .scan_sch_prio_tbl_cmdid = WMI_CMD_UNSUPPORTED, .scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_regdomain_cmdid = WMI_10_2_PDEV_SET_REGDOMAIN_CMDID, .pdev_set_channel_cmdid = WMI_10_2_PDEV_SET_CHANNEL_CMDID, .pdev_set_param_cmdid = WMI_10_2_PDEV_SET_PARAM_CMDID, .pdev_pktlog_enable_cmdid = WMI_10_2_PDEV_PKTLOG_ENABLE_CMDID, .pdev_pktlog_disable_cmdid = WMI_10_2_PDEV_PKTLOG_DISABLE_CMDID, .pdev_set_wmm_params_cmdid = WMI_10_2_PDEV_SET_WMM_PARAMS_CMDID, .pdev_set_ht_cap_ie_cmdid = WMI_10_2_PDEV_SET_HT_CAP_IE_CMDID, .pdev_set_vht_cap_ie_cmdid = WMI_10_2_PDEV_SET_VHT_CAP_IE_CMDID, .pdev_set_quiet_mode_cmdid = WMI_10_2_PDEV_SET_QUIET_MODE_CMDID, .pdev_green_ap_ps_enable_cmdid = WMI_10_2_PDEV_GREEN_AP_PS_ENABLE_CMDID, .pdev_get_tpc_config_cmdid = WMI_10_2_PDEV_GET_TPC_CONFIG_CMDID, .pdev_set_base_macaddr_cmdid = WMI_10_2_PDEV_SET_BASE_MACADDR_CMDID, .vdev_create_cmdid = WMI_10_2_VDEV_CREATE_CMDID, .vdev_delete_cmdid = WMI_10_2_VDEV_DELETE_CMDID, .vdev_start_request_cmdid = WMI_10_2_VDEV_START_REQUEST_CMDID, .vdev_restart_request_cmdid = WMI_10_2_VDEV_RESTART_REQUEST_CMDID, .vdev_up_cmdid = WMI_10_2_VDEV_UP_CMDID, .vdev_stop_cmdid = WMI_10_2_VDEV_STOP_CMDID, .vdev_down_cmdid = WMI_10_2_VDEV_DOWN_CMDID, .vdev_set_param_cmdid = WMI_10_2_VDEV_SET_PARAM_CMDID, .vdev_install_key_cmdid = WMI_10_2_VDEV_INSTALL_KEY_CMDID, .peer_create_cmdid = WMI_10_2_PEER_CREATE_CMDID, .peer_delete_cmdid = WMI_10_2_PEER_DELETE_CMDID, .peer_flush_tids_cmdid = WMI_10_2_PEER_FLUSH_TIDS_CMDID, .peer_set_param_cmdid = WMI_10_2_PEER_SET_PARAM_CMDID, .peer_assoc_cmdid = WMI_10_2_PEER_ASSOC_CMDID, .peer_add_wds_entry_cmdid = WMI_10_2_PEER_ADD_WDS_ENTRY_CMDID, .peer_remove_wds_entry_cmdid = WMI_10_2_PEER_REMOVE_WDS_ENTRY_CMDID, .peer_mcast_group_cmdid = WMI_10_2_PEER_MCAST_GROUP_CMDID, .bcn_tx_cmdid = WMI_10_2_BCN_TX_CMDID, .pdev_send_bcn_cmdid = WMI_10_2_PDEV_SEND_BCN_CMDID, .bcn_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .bcn_filter_rx_cmdid = WMI_10_2_BCN_FILTER_RX_CMDID, .prb_req_filter_rx_cmdid = WMI_10_2_PRB_REQ_FILTER_RX_CMDID, .mgmt_tx_cmdid = WMI_10_2_MGMT_TX_CMDID, .prb_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .addba_clear_resp_cmdid = WMI_10_2_ADDBA_CLEAR_RESP_CMDID, .addba_send_cmdid = WMI_10_2_ADDBA_SEND_CMDID, .addba_status_cmdid = WMI_10_2_ADDBA_STATUS_CMDID, .delba_send_cmdid = WMI_10_2_DELBA_SEND_CMDID, .addba_set_resp_cmdid = WMI_10_2_ADDBA_SET_RESP_CMDID, .send_singleamsdu_cmdid = WMI_10_2_SEND_SINGLEAMSDU_CMDID, .sta_powersave_mode_cmdid = WMI_10_2_STA_POWERSAVE_MODE_CMDID, .sta_powersave_param_cmdid = WMI_10_2_STA_POWERSAVE_PARAM_CMDID, .sta_mimo_ps_mode_cmdid = WMI_10_2_STA_MIMO_PS_MODE_CMDID, .pdev_dfs_enable_cmdid = WMI_10_2_PDEV_DFS_ENABLE_CMDID, .pdev_dfs_disable_cmdid = WMI_10_2_PDEV_DFS_DISABLE_CMDID, .roam_scan_mode = WMI_10_2_ROAM_SCAN_MODE, .roam_scan_rssi_threshold = WMI_10_2_ROAM_SCAN_RSSI_THRESHOLD, .roam_scan_period = WMI_10_2_ROAM_SCAN_PERIOD, .roam_scan_rssi_change_threshold = WMI_10_2_ROAM_SCAN_RSSI_CHANGE_THRESHOLD, .roam_ap_profile = WMI_10_2_ROAM_AP_PROFILE, .ofl_scan_add_ap_profile = WMI_10_2_OFL_SCAN_ADD_AP_PROFILE, .ofl_scan_remove_ap_profile = WMI_10_2_OFL_SCAN_REMOVE_AP_PROFILE, .ofl_scan_period = WMI_10_2_OFL_SCAN_PERIOD, .p2p_dev_set_device_info = WMI_10_2_P2P_DEV_SET_DEVICE_INFO, .p2p_dev_set_discoverability = WMI_10_2_P2P_DEV_SET_DISCOVERABILITY, .p2p_go_set_beacon_ie = WMI_10_2_P2P_GO_SET_BEACON_IE, .p2p_go_set_probe_resp_ie = WMI_10_2_P2P_GO_SET_PROBE_RESP_IE, .p2p_set_vendor_ie_data_cmdid = WMI_CMD_UNSUPPORTED, .ap_ps_peer_param_cmdid = WMI_10_2_AP_PS_PEER_PARAM_CMDID, .ap_ps_peer_uapsd_coex_cmdid = WMI_CMD_UNSUPPORTED, .peer_rate_retry_sched_cmdid = WMI_10_2_PEER_RATE_RETRY_SCHED_CMDID, .wlan_profile_trigger_cmdid = WMI_10_2_WLAN_PROFILE_TRIGGER_CMDID, .wlan_profile_set_hist_intvl_cmdid = WMI_10_2_WLAN_PROFILE_SET_HIST_INTVL_CMDID, .wlan_profile_get_profile_data_cmdid = WMI_10_2_WLAN_PROFILE_GET_PROFILE_DATA_CMDID, .wlan_profile_enable_profile_id_cmdid = WMI_10_2_WLAN_PROFILE_ENABLE_PROFILE_ID_CMDID, .wlan_profile_list_profile_id_cmdid = WMI_10_2_WLAN_PROFILE_LIST_PROFILE_ID_CMDID, .pdev_suspend_cmdid = WMI_10_2_PDEV_SUSPEND_CMDID, .pdev_resume_cmdid = WMI_10_2_PDEV_RESUME_CMDID, .add_bcn_filter_cmdid = WMI_10_2_ADD_BCN_FILTER_CMDID, .rmv_bcn_filter_cmdid = WMI_10_2_RMV_BCN_FILTER_CMDID, .wow_add_wake_pattern_cmdid = WMI_10_2_WOW_ADD_WAKE_PATTERN_CMDID, .wow_del_wake_pattern_cmdid = WMI_10_2_WOW_DEL_WAKE_PATTERN_CMDID, .wow_enable_disable_wake_event_cmdid = WMI_10_2_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID, .wow_enable_cmdid = WMI_10_2_WOW_ENABLE_CMDID, .wow_hostwakeup_from_sleep_cmdid = WMI_10_2_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID, .rtt_measreq_cmdid = WMI_10_2_RTT_MEASREQ_CMDID, .rtt_tsf_cmdid = WMI_10_2_RTT_TSF_CMDID, .vdev_spectral_scan_configure_cmdid = WMI_10_2_VDEV_SPECTRAL_SCAN_CONFIGURE_CMDID, .vdev_spectral_scan_enable_cmdid = WMI_10_2_VDEV_SPECTRAL_SCAN_ENABLE_CMDID, .request_stats_cmdid = WMI_10_2_REQUEST_STATS_CMDID, .set_arp_ns_offload_cmdid = WMI_CMD_UNSUPPORTED, .network_list_offload_config_cmdid = WMI_CMD_UNSUPPORTED, .gtk_offload_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_enable_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_chanswitch_cmdid = WMI_CMD_UNSUPPORTED, .chatter_set_mode_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_addba_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_delba_cmdid = WMI_CMD_UNSUPPORTED, .sta_dtim_ps_method_cmdid = WMI_CMD_UNSUPPORTED, .sta_uapsd_auto_trig_cmdid = WMI_CMD_UNSUPPORTED, .sta_keepalive_cmd = WMI_CMD_UNSUPPORTED, .echo_cmdid = WMI_10_2_ECHO_CMDID, .pdev_utf_cmdid = WMI_10_2_PDEV_UTF_CMDID, .dbglog_cfg_cmdid = WMI_10_2_DBGLOG_CFG_CMDID, .pdev_qvit_cmdid = WMI_10_2_PDEV_QVIT_CMDID, .pdev_ftm_intg_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .force_fw_hang_cmdid = WMI_CMD_UNSUPPORTED, .gpio_config_cmdid = WMI_10_2_GPIO_CONFIG_CMDID, .gpio_output_cmdid = WMI_10_2_GPIO_OUTPUT_CMDID, .pdev_get_temperature_cmdid = WMI_10_2_PDEV_GET_TEMPERATURE_CMDID, .pdev_enable_adaptive_cca_cmdid = WMI_10_2_SET_CCA_PARAMS, .scan_update_request_cmdid = WMI_CMD_UNSUPPORTED, .vdev_standby_response_cmdid = WMI_CMD_UNSUPPORTED, .vdev_resume_response_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_add_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_evict_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_restore_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_print_all_peers_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_update_wds_entry_cmdid = WMI_CMD_UNSUPPORTED, .peer_add_proxy_sta_entry_cmdid = WMI_CMD_UNSUPPORTED, .rtt_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .oem_req_cmdid = WMI_CMD_UNSUPPORTED, .nan_cmdid = WMI_CMD_UNSUPPORTED, .vdev_ratemask_cmdid = WMI_CMD_UNSUPPORTED, .qboost_cfg_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_enable_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_set_rx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_tx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_train_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_node_config_ops_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_antenna_switch_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_ctl_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_mimogain_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_chainmsk_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_fips_cmdid = WMI_CMD_UNSUPPORTED, .tt_set_conf_cmdid = WMI_CMD_UNSUPPORTED, .fwtest_cmdid = WMI_CMD_UNSUPPORTED, .vdev_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .peer_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_cck_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_ofdm_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_reserve_ast_entry_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_nfcal_power_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ast_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_dscp_tid_map_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_info_cmdid = WMI_CMD_UNSUPPORTED, .vdev_filter_neighbor_rx_packets_cmdid = WMI_CMD_UNSUPPORTED, .mu_cal_start_cmdid = WMI_CMD_UNSUPPORTED, .set_cca_params_cmdid = WMI_CMD_UNSUPPORTED, .pdev_bss_chan_info_request_cmdid = WMI_10_2_PDEV_BSS_CHAN_INFO_REQUEST_CMDID, .pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED, .radar_found_cmdid = WMI_CMD_UNSUPPORTED, .set_bb_timing_cmdid = WMI_10_2_PDEV_SET_BB_TIMING_CONFIG_CMDID, }; /* 10.4 WMI cmd track */ static struct wmi_cmd_map wmi_10_4_cmd_map = { .init_cmdid = WMI_10_4_INIT_CMDID, .start_scan_cmdid = WMI_10_4_START_SCAN_CMDID, .stop_scan_cmdid = WMI_10_4_STOP_SCAN_CMDID, .scan_chan_list_cmdid = WMI_10_4_SCAN_CHAN_LIST_CMDID, .scan_sch_prio_tbl_cmdid = WMI_10_4_SCAN_SCH_PRIO_TBL_CMDID, .scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_regdomain_cmdid = WMI_10_4_PDEV_SET_REGDOMAIN_CMDID, .pdev_set_channel_cmdid = WMI_10_4_PDEV_SET_CHANNEL_CMDID, .pdev_set_param_cmdid = WMI_10_4_PDEV_SET_PARAM_CMDID, .pdev_pktlog_enable_cmdid = WMI_10_4_PDEV_PKTLOG_ENABLE_CMDID, .pdev_pktlog_disable_cmdid = WMI_10_4_PDEV_PKTLOG_DISABLE_CMDID, .pdev_set_wmm_params_cmdid = WMI_10_4_PDEV_SET_WMM_PARAMS_CMDID, .pdev_set_ht_cap_ie_cmdid = WMI_10_4_PDEV_SET_HT_CAP_IE_CMDID, .pdev_set_vht_cap_ie_cmdid = WMI_10_4_PDEV_SET_VHT_CAP_IE_CMDID, .pdev_set_dscp_tid_map_cmdid = WMI_10_4_PDEV_SET_DSCP_TID_MAP_CMDID, .pdev_set_quiet_mode_cmdid = WMI_10_4_PDEV_SET_QUIET_MODE_CMDID, .pdev_green_ap_ps_enable_cmdid = WMI_10_4_PDEV_GREEN_AP_PS_ENABLE_CMDID, .pdev_get_tpc_config_cmdid = WMI_10_4_PDEV_GET_TPC_CONFIG_CMDID, .pdev_set_base_macaddr_cmdid = WMI_10_4_PDEV_SET_BASE_MACADDR_CMDID, .vdev_create_cmdid = WMI_10_4_VDEV_CREATE_CMDID, .vdev_delete_cmdid = WMI_10_4_VDEV_DELETE_CMDID, .vdev_start_request_cmdid = WMI_10_4_VDEV_START_REQUEST_CMDID, .vdev_restart_request_cmdid = WMI_10_4_VDEV_RESTART_REQUEST_CMDID, .vdev_up_cmdid = WMI_10_4_VDEV_UP_CMDID, .vdev_stop_cmdid = WMI_10_4_VDEV_STOP_CMDID, .vdev_down_cmdid = WMI_10_4_VDEV_DOWN_CMDID, .vdev_set_param_cmdid = WMI_10_4_VDEV_SET_PARAM_CMDID, .vdev_install_key_cmdid = WMI_10_4_VDEV_INSTALL_KEY_CMDID, .peer_create_cmdid = WMI_10_4_PEER_CREATE_CMDID, .peer_delete_cmdid = WMI_10_4_PEER_DELETE_CMDID, .peer_flush_tids_cmdid = WMI_10_4_PEER_FLUSH_TIDS_CMDID, .peer_set_param_cmdid = WMI_10_4_PEER_SET_PARAM_CMDID, .peer_assoc_cmdid = WMI_10_4_PEER_ASSOC_CMDID, .peer_add_wds_entry_cmdid = WMI_10_4_PEER_ADD_WDS_ENTRY_CMDID, .peer_remove_wds_entry_cmdid = WMI_10_4_PEER_REMOVE_WDS_ENTRY_CMDID, .peer_mcast_group_cmdid = WMI_10_4_PEER_MCAST_GROUP_CMDID, .bcn_tx_cmdid = WMI_10_4_BCN_TX_CMDID, .pdev_send_bcn_cmdid = WMI_10_4_PDEV_SEND_BCN_CMDID, .bcn_tmpl_cmdid = WMI_10_4_BCN_PRB_TMPL_CMDID, .bcn_filter_rx_cmdid = WMI_10_4_BCN_FILTER_RX_CMDID, .prb_req_filter_rx_cmdid = WMI_10_4_PRB_REQ_FILTER_RX_CMDID, .mgmt_tx_cmdid = WMI_10_4_MGMT_TX_CMDID, .prb_tmpl_cmdid = WMI_10_4_PRB_TMPL_CMDID, .addba_clear_resp_cmdid = WMI_10_4_ADDBA_CLEAR_RESP_CMDID, .addba_send_cmdid = WMI_10_4_ADDBA_SEND_CMDID, .addba_status_cmdid = WMI_10_4_ADDBA_STATUS_CMDID, .delba_send_cmdid = WMI_10_4_DELBA_SEND_CMDID, .addba_set_resp_cmdid = WMI_10_4_ADDBA_SET_RESP_CMDID, .send_singleamsdu_cmdid = WMI_10_4_SEND_SINGLEAMSDU_CMDID, .sta_powersave_mode_cmdid = WMI_10_4_STA_POWERSAVE_MODE_CMDID, .sta_powersave_param_cmdid = WMI_10_4_STA_POWERSAVE_PARAM_CMDID, .sta_mimo_ps_mode_cmdid = WMI_10_4_STA_MIMO_PS_MODE_CMDID, .pdev_dfs_enable_cmdid = WMI_10_4_PDEV_DFS_ENABLE_CMDID, .pdev_dfs_disable_cmdid = WMI_10_4_PDEV_DFS_DISABLE_CMDID, .roam_scan_mode = WMI_10_4_ROAM_SCAN_MODE, .roam_scan_rssi_threshold = WMI_10_4_ROAM_SCAN_RSSI_THRESHOLD, .roam_scan_period = WMI_10_4_ROAM_SCAN_PERIOD, .roam_scan_rssi_change_threshold = WMI_10_4_ROAM_SCAN_RSSI_CHANGE_THRESHOLD, .roam_ap_profile = WMI_10_4_ROAM_AP_PROFILE, .ofl_scan_add_ap_profile = WMI_10_4_OFL_SCAN_ADD_AP_PROFILE, .ofl_scan_remove_ap_profile = WMI_10_4_OFL_SCAN_REMOVE_AP_PROFILE, .ofl_scan_period = WMI_10_4_OFL_SCAN_PERIOD, .p2p_dev_set_device_info = WMI_10_4_P2P_DEV_SET_DEVICE_INFO, .p2p_dev_set_discoverability = WMI_10_4_P2P_DEV_SET_DISCOVERABILITY, .p2p_go_set_beacon_ie = WMI_10_4_P2P_GO_SET_BEACON_IE, .p2p_go_set_probe_resp_ie = WMI_10_4_P2P_GO_SET_PROBE_RESP_IE, .p2p_set_vendor_ie_data_cmdid = WMI_10_4_P2P_SET_VENDOR_IE_DATA_CMDID, .ap_ps_peer_param_cmdid = WMI_10_4_AP_PS_PEER_PARAM_CMDID, .ap_ps_peer_uapsd_coex_cmdid = WMI_10_4_AP_PS_PEER_UAPSD_COEX_CMDID, .peer_rate_retry_sched_cmdid = WMI_10_4_PEER_RATE_RETRY_SCHED_CMDID, .wlan_profile_trigger_cmdid = WMI_10_4_WLAN_PROFILE_TRIGGER_CMDID, .wlan_profile_set_hist_intvl_cmdid = WMI_10_4_WLAN_PROFILE_SET_HIST_INTVL_CMDID, .wlan_profile_get_profile_data_cmdid = WMI_10_4_WLAN_PROFILE_GET_PROFILE_DATA_CMDID, .wlan_profile_enable_profile_id_cmdid = WMI_10_4_WLAN_PROFILE_ENABLE_PROFILE_ID_CMDID, .wlan_profile_list_profile_id_cmdid = WMI_10_4_WLAN_PROFILE_LIST_PROFILE_ID_CMDID, .pdev_suspend_cmdid = WMI_10_4_PDEV_SUSPEND_CMDID, .pdev_resume_cmdid = WMI_10_4_PDEV_RESUME_CMDID, .add_bcn_filter_cmdid = WMI_10_4_ADD_BCN_FILTER_CMDID, .rmv_bcn_filter_cmdid = WMI_10_4_RMV_BCN_FILTER_CMDID, .wow_add_wake_pattern_cmdid = WMI_10_4_WOW_ADD_WAKE_PATTERN_CMDID, .wow_del_wake_pattern_cmdid = WMI_10_4_WOW_DEL_WAKE_PATTERN_CMDID, .wow_enable_disable_wake_event_cmdid = WMI_10_4_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID, .wow_enable_cmdid = WMI_10_4_WOW_ENABLE_CMDID, .wow_hostwakeup_from_sleep_cmdid = WMI_10_4_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID, .rtt_measreq_cmdid = WMI_10_4_RTT_MEASREQ_CMDID, .rtt_tsf_cmdid = WMI_10_4_RTT_TSF_CMDID, .vdev_spectral_scan_configure_cmdid = WMI_10_4_VDEV_SPECTRAL_SCAN_CONFIGURE_CMDID, .vdev_spectral_scan_enable_cmdid = WMI_10_4_VDEV_SPECTRAL_SCAN_ENABLE_CMDID, .request_stats_cmdid = WMI_10_4_REQUEST_STATS_CMDID, .set_arp_ns_offload_cmdid = WMI_CMD_UNSUPPORTED, .network_list_offload_config_cmdid = WMI_CMD_UNSUPPORTED, .gtk_offload_cmdid = WMI_10_4_GTK_OFFLOAD_CMDID, .csa_offload_enable_cmdid = WMI_10_4_CSA_OFFLOAD_ENABLE_CMDID, .csa_offload_chanswitch_cmdid = WMI_10_4_CSA_OFFLOAD_CHANSWITCH_CMDID, .chatter_set_mode_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_addba_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_delba_cmdid = WMI_CMD_UNSUPPORTED, .sta_dtim_ps_method_cmdid = WMI_CMD_UNSUPPORTED, .sta_uapsd_auto_trig_cmdid = WMI_CMD_UNSUPPORTED, .sta_keepalive_cmd = WMI_CMD_UNSUPPORTED, .echo_cmdid = WMI_10_4_ECHO_CMDID, .pdev_utf_cmdid = WMI_10_4_PDEV_UTF_CMDID, .dbglog_cfg_cmdid = WMI_10_4_DBGLOG_CFG_CMDID, .pdev_qvit_cmdid = WMI_10_4_PDEV_QVIT_CMDID, .pdev_ftm_intg_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_keepalive_cmdid = WMI_10_4_VDEV_SET_KEEPALIVE_CMDID, .vdev_get_keepalive_cmdid = WMI_10_4_VDEV_GET_KEEPALIVE_CMDID, .force_fw_hang_cmdid = WMI_10_4_FORCE_FW_HANG_CMDID, .gpio_config_cmdid = WMI_10_4_GPIO_CONFIG_CMDID, .gpio_output_cmdid = WMI_10_4_GPIO_OUTPUT_CMDID, .pdev_get_temperature_cmdid = WMI_10_4_PDEV_GET_TEMPERATURE_CMDID, .vdev_set_wmm_params_cmdid = WMI_CMD_UNSUPPORTED, .adaptive_qcs_cmdid = WMI_CMD_UNSUPPORTED, .scan_update_request_cmdid = WMI_10_4_SCAN_UPDATE_REQUEST_CMDID, .vdev_standby_response_cmdid = WMI_10_4_VDEV_STANDBY_RESPONSE_CMDID, .vdev_resume_response_cmdid = WMI_10_4_VDEV_RESUME_RESPONSE_CMDID, .wlan_peer_caching_add_peer_cmdid = WMI_10_4_WLAN_PEER_CACHING_ADD_PEER_CMDID, .wlan_peer_caching_evict_peer_cmdid = WMI_10_4_WLAN_PEER_CACHING_EVICT_PEER_CMDID, .wlan_peer_caching_restore_peer_cmdid = WMI_10_4_WLAN_PEER_CACHING_RESTORE_PEER_CMDID, .wlan_peer_caching_print_all_peers_info_cmdid = WMI_10_4_WLAN_PEER_CACHING_PRINT_ALL_PEERS_INFO_CMDID, .peer_update_wds_entry_cmdid = WMI_10_4_PEER_UPDATE_WDS_ENTRY_CMDID, .peer_add_proxy_sta_entry_cmdid = WMI_10_4_PEER_ADD_PROXY_STA_ENTRY_CMDID, .rtt_keepalive_cmdid = WMI_10_4_RTT_KEEPALIVE_CMDID, .oem_req_cmdid = WMI_10_4_OEM_REQ_CMDID, .nan_cmdid = WMI_10_4_NAN_CMDID, .vdev_ratemask_cmdid = WMI_10_4_VDEV_RATEMASK_CMDID, .qboost_cfg_cmdid = WMI_10_4_QBOOST_CFG_CMDID, .pdev_smart_ant_enable_cmdid = WMI_10_4_PDEV_SMART_ANT_ENABLE_CMDID, .pdev_smart_ant_set_rx_antenna_cmdid = WMI_10_4_PDEV_SMART_ANT_SET_RX_ANTENNA_CMDID, .peer_smart_ant_set_tx_antenna_cmdid = WMI_10_4_PEER_SMART_ANT_SET_TX_ANTENNA_CMDID, .peer_smart_ant_set_train_info_cmdid = WMI_10_4_PEER_SMART_ANT_SET_TRAIN_INFO_CMDID, .peer_smart_ant_set_node_config_ops_cmdid = WMI_10_4_PEER_SMART_ANT_SET_NODE_CONFIG_OPS_CMDID, .pdev_set_antenna_switch_table_cmdid = WMI_10_4_PDEV_SET_ANTENNA_SWITCH_TABLE_CMDID, .pdev_set_ctl_table_cmdid = WMI_10_4_PDEV_SET_CTL_TABLE_CMDID, .pdev_set_mimogain_table_cmdid = WMI_10_4_PDEV_SET_MIMOGAIN_TABLE_CMDID, .pdev_ratepwr_table_cmdid = WMI_10_4_PDEV_RATEPWR_TABLE_CMDID, .pdev_ratepwr_chainmsk_table_cmdid = WMI_10_4_PDEV_RATEPWR_CHAINMSK_TABLE_CMDID, .pdev_fips_cmdid = WMI_10_4_PDEV_FIPS_CMDID, .tt_set_conf_cmdid = WMI_10_4_TT_SET_CONF_CMDID, .fwtest_cmdid = WMI_10_4_FWTEST_CMDID, .vdev_atf_request_cmdid = WMI_10_4_VDEV_ATF_REQUEST_CMDID, .peer_atf_request_cmdid = WMI_10_4_PEER_ATF_REQUEST_CMDID, .pdev_get_ani_cck_config_cmdid = WMI_10_4_PDEV_GET_ANI_CCK_CONFIG_CMDID, .pdev_get_ani_ofdm_config_cmdid = WMI_10_4_PDEV_GET_ANI_OFDM_CONFIG_CMDID, .pdev_reserve_ast_entry_cmdid = WMI_10_4_PDEV_RESERVE_AST_ENTRY_CMDID, .pdev_get_nfcal_power_cmdid = WMI_10_4_PDEV_GET_NFCAL_POWER_CMDID, .pdev_get_tpc_cmdid = WMI_10_4_PDEV_GET_TPC_CMDID, .pdev_get_ast_info_cmdid = WMI_10_4_PDEV_GET_AST_INFO_CMDID, .vdev_set_dscp_tid_map_cmdid = WMI_10_4_VDEV_SET_DSCP_TID_MAP_CMDID, .pdev_get_info_cmdid = WMI_10_4_PDEV_GET_INFO_CMDID, .vdev_get_info_cmdid = WMI_10_4_VDEV_GET_INFO_CMDID, .vdev_filter_neighbor_rx_packets_cmdid = WMI_10_4_VDEV_FILTER_NEIGHBOR_RX_PACKETS_CMDID, .mu_cal_start_cmdid = WMI_10_4_MU_CAL_START_CMDID, .set_cca_params_cmdid = WMI_10_4_SET_CCA_PARAMS_CMDID, .pdev_bss_chan_info_request_cmdid = WMI_10_4_PDEV_BSS_CHAN_INFO_REQUEST_CMDID, .ext_resource_cfg_cmdid = WMI_10_4_EXT_RESOURCE_CFG_CMDID, .vdev_set_ie_cmdid = WMI_10_4_VDEV_SET_IE_CMDID, .set_lteu_config_cmdid = WMI_10_4_SET_LTEU_CONFIG_CMDID, .atf_ssid_grouping_request_cmdid = WMI_10_4_ATF_SSID_GROUPING_REQUEST_CMDID, .peer_atf_ext_request_cmdid = WMI_10_4_PEER_ATF_EXT_REQUEST_CMDID, .set_periodic_channel_stats_cfg_cmdid = WMI_10_4_SET_PERIODIC_CHANNEL_STATS_CONFIG, .peer_bwf_request_cmdid = WMI_10_4_PEER_BWF_REQUEST_CMDID, .btcoex_cfg_cmdid = WMI_10_4_BTCOEX_CFG_CMDID, .peer_tx_mu_txmit_count_cmdid = WMI_10_4_PEER_TX_MU_TXMIT_COUNT_CMDID, .peer_tx_mu_txmit_rstcnt_cmdid = WMI_10_4_PEER_TX_MU_TXMIT_RSTCNT_CMDID, .peer_gid_userpos_list_cmdid = WMI_10_4_PEER_GID_USERPOS_LIST_CMDID, .pdev_check_cal_version_cmdid = WMI_10_4_PDEV_CHECK_CAL_VERSION_CMDID, .coex_version_cfg_cmid = WMI_10_4_COEX_VERSION_CFG_CMID, .pdev_get_rx_filter_cmdid = WMI_10_4_PDEV_GET_RX_FILTER_CMDID, .pdev_extended_nss_cfg_cmdid = WMI_10_4_PDEV_EXTENDED_NSS_CFG_CMDID, .vdev_set_scan_nac_rssi_cmdid = WMI_10_4_VDEV_SET_SCAN_NAC_RSSI_CMDID, .prog_gpio_band_select_cmdid = WMI_10_4_PROG_GPIO_BAND_SELECT_CMDID, .config_smart_logging_cmdid = WMI_10_4_CONFIG_SMART_LOGGING_CMDID, .debug_fatal_condition_cmdid = WMI_10_4_DEBUG_FATAL_CONDITION_CMDID, .get_tsf_timer_cmdid = WMI_10_4_GET_TSF_TIMER_CMDID, .pdev_get_tpc_table_cmdid = WMI_10_4_PDEV_GET_TPC_TABLE_CMDID, .vdev_sifs_trigger_time_cmdid = WMI_10_4_VDEV_SIFS_TRIGGER_TIME_CMDID, .pdev_wds_entry_list_cmdid = WMI_10_4_PDEV_WDS_ENTRY_LIST_CMDID, .tdls_set_state_cmdid = WMI_10_4_TDLS_SET_STATE_CMDID, .tdls_peer_update_cmdid = WMI_10_4_TDLS_PEER_UPDATE_CMDID, .tdls_set_offchan_mode_cmdid = WMI_10_4_TDLS_SET_OFFCHAN_MODE_CMDID, .radar_found_cmdid = WMI_10_4_RADAR_FOUND_CMDID, .per_peer_per_tid_config_cmdid = WMI_10_4_PER_PEER_PER_TID_CONFIG_CMDID, }; static struct wmi_peer_param_map wmi_peer_param_map = { .smps_state = WMI_PEER_SMPS_STATE, .ampdu = WMI_PEER_AMPDU, .authorize = WMI_PEER_AUTHORIZE, .chan_width = WMI_PEER_CHAN_WIDTH, .nss = WMI_PEER_NSS, .use_4addr = WMI_PEER_USE_4ADDR, .use_fixed_power = WMI_PEER_USE_FIXED_PWR, .debug = WMI_PEER_DEBUG, .phymode = WMI_PEER_PHYMODE, .dummy_var = WMI_PEER_DUMMY_VAR, }; /* MAIN WMI VDEV param map */ static struct wmi_vdev_param_map wmi_vdev_param_map = { .rts_threshold = WMI_VDEV_PARAM_RTS_THRESHOLD, .fragmentation_threshold = WMI_VDEV_PARAM_FRAGMENTATION_THRESHOLD, .beacon_interval = WMI_VDEV_PARAM_BEACON_INTERVAL, .listen_interval = WMI_VDEV_PARAM_LISTEN_INTERVAL, .multicast_rate = WMI_VDEV_PARAM_MULTICAST_RATE, .mgmt_tx_rate = WMI_VDEV_PARAM_MGMT_TX_RATE, .slot_time = WMI_VDEV_PARAM_SLOT_TIME, .preamble = WMI_VDEV_PARAM_PREAMBLE, .swba_time = WMI_VDEV_PARAM_SWBA_TIME, .wmi_vdev_stats_update_period = WMI_VDEV_STATS_UPDATE_PERIOD, .wmi_vdev_pwrsave_ageout_time = WMI_VDEV_PWRSAVE_AGEOUT_TIME, .wmi_vdev_host_swba_interval = WMI_VDEV_HOST_SWBA_INTERVAL, .dtim_period = WMI_VDEV_PARAM_DTIM_PERIOD, .wmi_vdev_oc_scheduler_air_time_limit = WMI_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT, .wds = WMI_VDEV_PARAM_WDS, .atim_window = WMI_VDEV_PARAM_ATIM_WINDOW, .bmiss_count_max = WMI_VDEV_PARAM_BMISS_COUNT_MAX, .bmiss_first_bcnt = WMI_VDEV_PARAM_BMISS_FIRST_BCNT, .bmiss_final_bcnt = WMI_VDEV_PARAM_BMISS_FINAL_BCNT, .feature_wmm = WMI_VDEV_PARAM_FEATURE_WMM, .chwidth = WMI_VDEV_PARAM_CHWIDTH, .chextoffset = WMI_VDEV_PARAM_CHEXTOFFSET, .disable_htprotection = WMI_VDEV_PARAM_DISABLE_HTPROTECTION, .sta_quickkickout = WMI_VDEV_PARAM_STA_QUICKKICKOUT, .mgmt_rate = WMI_VDEV_PARAM_MGMT_RATE, .protection_mode = WMI_VDEV_PARAM_PROTECTION_MODE, .fixed_rate = WMI_VDEV_PARAM_FIXED_RATE, .sgi = WMI_VDEV_PARAM_SGI, .ldpc = WMI_VDEV_PARAM_LDPC, .tx_stbc = WMI_VDEV_PARAM_TX_STBC, .rx_stbc = WMI_VDEV_PARAM_RX_STBC, .intra_bss_fwd = WMI_VDEV_PARAM_INTRA_BSS_FWD, .def_keyid = WMI_VDEV_PARAM_DEF_KEYID, .nss = WMI_VDEV_PARAM_NSS, .bcast_data_rate = WMI_VDEV_PARAM_BCAST_DATA_RATE, .mcast_data_rate = WMI_VDEV_PARAM_MCAST_DATA_RATE, .mcast_indicate = WMI_VDEV_PARAM_MCAST_INDICATE, .dhcp_indicate = WMI_VDEV_PARAM_DHCP_INDICATE, .unknown_dest_indicate = WMI_VDEV_PARAM_UNKNOWN_DEST_INDICATE, .ap_keepalive_min_idle_inactive_time_secs = WMI_VDEV_PARAM_AP_KEEPALIVE_MIN_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_idle_inactive_time_secs = WMI_VDEV_PARAM_AP_KEEPALIVE_MAX_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_unresponsive_time_secs = WMI_VDEV_PARAM_AP_KEEPALIVE_MAX_UNRESPONSIVE_TIME_SECS, .ap_enable_nawds = WMI_VDEV_PARAM_AP_ENABLE_NAWDS, .mcast2ucast_set = WMI_VDEV_PARAM_UNSUPPORTED, .enable_rtscts = WMI_VDEV_PARAM_ENABLE_RTSCTS, .txbf = WMI_VDEV_PARAM_TXBF, .packet_powersave = WMI_VDEV_PARAM_PACKET_POWERSAVE, .drop_unencry = WMI_VDEV_PARAM_DROP_UNENCRY, .tx_encap_type = WMI_VDEV_PARAM_TX_ENCAP_TYPE, .ap_detect_out_of_sync_sleeping_sta_time_secs = WMI_VDEV_PARAM_UNSUPPORTED, .rc_num_retries = WMI_VDEV_PARAM_UNSUPPORTED, .cabq_maxdur = WMI_VDEV_PARAM_UNSUPPORTED, .mfptest_set = WMI_VDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_VDEV_PARAM_UNSUPPORTED, .vht_sgimask = WMI_VDEV_PARAM_UNSUPPORTED, .vht80_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_enable = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_tgt_bmiss_num = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_bmiss_sample_cycle = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_slop_step = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_init_slop = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_pause = WMI_VDEV_PARAM_UNSUPPORTED, .proxy_sta = WMI_VDEV_PARAM_UNSUPPORTED, .meru_vc = WMI_VDEV_PARAM_UNSUPPORTED, .rx_decap_type = WMI_VDEV_PARAM_UNSUPPORTED, .bw_nss_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .disable_4addr_src_lrn = WMI_VDEV_PARAM_UNSUPPORTED, .rtt_responder_role = WMI_VDEV_PARAM_UNSUPPORTED, }; /* 10.X WMI VDEV param map */ static struct wmi_vdev_param_map wmi_10x_vdev_param_map = { .rts_threshold = WMI_10X_VDEV_PARAM_RTS_THRESHOLD, .fragmentation_threshold = WMI_10X_VDEV_PARAM_FRAGMENTATION_THRESHOLD, .beacon_interval = WMI_10X_VDEV_PARAM_BEACON_INTERVAL, .listen_interval = WMI_10X_VDEV_PARAM_LISTEN_INTERVAL, .multicast_rate = WMI_10X_VDEV_PARAM_MULTICAST_RATE, .mgmt_tx_rate = WMI_10X_VDEV_PARAM_MGMT_TX_RATE, .slot_time = WMI_10X_VDEV_PARAM_SLOT_TIME, .preamble = WMI_10X_VDEV_PARAM_PREAMBLE, .swba_time = WMI_10X_VDEV_PARAM_SWBA_TIME, .wmi_vdev_stats_update_period = WMI_10X_VDEV_STATS_UPDATE_PERIOD, .wmi_vdev_pwrsave_ageout_time = WMI_10X_VDEV_PWRSAVE_AGEOUT_TIME, .wmi_vdev_host_swba_interval = WMI_10X_VDEV_HOST_SWBA_INTERVAL, .dtim_period = WMI_10X_VDEV_PARAM_DTIM_PERIOD, .wmi_vdev_oc_scheduler_air_time_limit = WMI_10X_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT, .wds = WMI_10X_VDEV_PARAM_WDS, .atim_window = WMI_10X_VDEV_PARAM_ATIM_WINDOW, .bmiss_count_max = WMI_10X_VDEV_PARAM_BMISS_COUNT_MAX, .bmiss_first_bcnt = WMI_VDEV_PARAM_UNSUPPORTED, .bmiss_final_bcnt = WMI_VDEV_PARAM_UNSUPPORTED, .feature_wmm = WMI_10X_VDEV_PARAM_FEATURE_WMM, .chwidth = WMI_10X_VDEV_PARAM_CHWIDTH, .chextoffset = WMI_10X_VDEV_PARAM_CHEXTOFFSET, .disable_htprotection = WMI_10X_VDEV_PARAM_DISABLE_HTPROTECTION, .sta_quickkickout = WMI_10X_VDEV_PARAM_STA_QUICKKICKOUT, .mgmt_rate = WMI_10X_VDEV_PARAM_MGMT_RATE, .protection_mode = WMI_10X_VDEV_PARAM_PROTECTION_MODE, .fixed_rate = WMI_10X_VDEV_PARAM_FIXED_RATE, .sgi = WMI_10X_VDEV_PARAM_SGI, .ldpc = WMI_10X_VDEV_PARAM_LDPC, .tx_stbc = WMI_10X_VDEV_PARAM_TX_STBC, .rx_stbc = WMI_10X_VDEV_PARAM_RX_STBC, .intra_bss_fwd = WMI_10X_VDEV_PARAM_INTRA_BSS_FWD, .def_keyid = WMI_10X_VDEV_PARAM_DEF_KEYID, .nss = WMI_10X_VDEV_PARAM_NSS, .bcast_data_rate = WMI_10X_VDEV_PARAM_BCAST_DATA_RATE, .mcast_data_rate = WMI_10X_VDEV_PARAM_MCAST_DATA_RATE, .mcast_indicate = WMI_10X_VDEV_PARAM_MCAST_INDICATE, .dhcp_indicate = WMI_10X_VDEV_PARAM_DHCP_INDICATE, .unknown_dest_indicate = WMI_10X_VDEV_PARAM_UNKNOWN_DEST_INDICATE, .ap_keepalive_min_idle_inactive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MIN_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_idle_inactive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MAX_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_unresponsive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MAX_UNRESPONSIVE_TIME_SECS, .ap_enable_nawds = WMI_10X_VDEV_PARAM_AP_ENABLE_NAWDS, .mcast2ucast_set = WMI_10X_VDEV_PARAM_MCAST2UCAST_SET, .enable_rtscts = WMI_10X_VDEV_PARAM_ENABLE_RTSCTS, .txbf = WMI_VDEV_PARAM_UNSUPPORTED, .packet_powersave = WMI_VDEV_PARAM_UNSUPPORTED, .drop_unencry = WMI_VDEV_PARAM_UNSUPPORTED, .tx_encap_type = WMI_VDEV_PARAM_UNSUPPORTED, .ap_detect_out_of_sync_sleeping_sta_time_secs = WMI_10X_VDEV_PARAM_AP_DETECT_OUT_OF_SYNC_SLEEPING_STA_TIME_SECS, .rc_num_retries = WMI_VDEV_PARAM_UNSUPPORTED, .cabq_maxdur = WMI_VDEV_PARAM_UNSUPPORTED, .mfptest_set = WMI_VDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_VDEV_PARAM_UNSUPPORTED, .vht_sgimask = WMI_VDEV_PARAM_UNSUPPORTED, .vht80_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_enable = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_tgt_bmiss_num = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_bmiss_sample_cycle = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_slop_step = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_init_slop = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_pause = WMI_VDEV_PARAM_UNSUPPORTED, .proxy_sta = WMI_VDEV_PARAM_UNSUPPORTED, .meru_vc = WMI_VDEV_PARAM_UNSUPPORTED, .rx_decap_type = WMI_VDEV_PARAM_UNSUPPORTED, .bw_nss_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .disable_4addr_src_lrn = WMI_VDEV_PARAM_UNSUPPORTED, .rtt_responder_role = WMI_VDEV_PARAM_UNSUPPORTED, }; static struct wmi_vdev_param_map wmi_10_2_4_vdev_param_map = { .rts_threshold = WMI_10X_VDEV_PARAM_RTS_THRESHOLD, .fragmentation_threshold = WMI_10X_VDEV_PARAM_FRAGMENTATION_THRESHOLD, .beacon_interval = WMI_10X_VDEV_PARAM_BEACON_INTERVAL, .listen_interval = WMI_10X_VDEV_PARAM_LISTEN_INTERVAL, .multicast_rate = WMI_10X_VDEV_PARAM_MULTICAST_RATE, .mgmt_tx_rate = WMI_10X_VDEV_PARAM_MGMT_TX_RATE, .slot_time = WMI_10X_VDEV_PARAM_SLOT_TIME, .preamble = WMI_10X_VDEV_PARAM_PREAMBLE, .swba_time = WMI_10X_VDEV_PARAM_SWBA_TIME, .wmi_vdev_stats_update_period = WMI_10X_VDEV_STATS_UPDATE_PERIOD, .wmi_vdev_pwrsave_ageout_time = WMI_10X_VDEV_PWRSAVE_AGEOUT_TIME, .wmi_vdev_host_swba_interval = WMI_10X_VDEV_HOST_SWBA_INTERVAL, .dtim_period = WMI_10X_VDEV_PARAM_DTIM_PERIOD, .wmi_vdev_oc_scheduler_air_time_limit = WMI_10X_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT, .wds = WMI_10X_VDEV_PARAM_WDS, .atim_window = WMI_10X_VDEV_PARAM_ATIM_WINDOW, .bmiss_count_max = WMI_10X_VDEV_PARAM_BMISS_COUNT_MAX, .bmiss_first_bcnt = WMI_VDEV_PARAM_UNSUPPORTED, .bmiss_final_bcnt = WMI_VDEV_PARAM_UNSUPPORTED, .feature_wmm = WMI_10X_VDEV_PARAM_FEATURE_WMM, .chwidth = WMI_10X_VDEV_PARAM_CHWIDTH, .chextoffset = WMI_10X_VDEV_PARAM_CHEXTOFFSET, .disable_htprotection = WMI_10X_VDEV_PARAM_DISABLE_HTPROTECTION, .sta_quickkickout = WMI_10X_VDEV_PARAM_STA_QUICKKICKOUT, .mgmt_rate = WMI_10X_VDEV_PARAM_MGMT_RATE, .protection_mode = WMI_10X_VDEV_PARAM_PROTECTION_MODE, .fixed_rate = WMI_10X_VDEV_PARAM_FIXED_RATE, .sgi = WMI_10X_VDEV_PARAM_SGI, .ldpc = WMI_10X_VDEV_PARAM_LDPC, .tx_stbc = WMI_10X_VDEV_PARAM_TX_STBC, .rx_stbc = WMI_10X_VDEV_PARAM_RX_STBC, .intra_bss_fwd = WMI_10X_VDEV_PARAM_INTRA_BSS_FWD, .def_keyid = WMI_10X_VDEV_PARAM_DEF_KEYID, .nss = WMI_10X_VDEV_PARAM_NSS, .bcast_data_rate = WMI_10X_VDEV_PARAM_BCAST_DATA_RATE, .mcast_data_rate = WMI_10X_VDEV_PARAM_MCAST_DATA_RATE, .mcast_indicate = WMI_10X_VDEV_PARAM_MCAST_INDICATE, .dhcp_indicate = WMI_10X_VDEV_PARAM_DHCP_INDICATE, .unknown_dest_indicate = WMI_10X_VDEV_PARAM_UNKNOWN_DEST_INDICATE, .ap_keepalive_min_idle_inactive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MIN_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_idle_inactive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MAX_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_unresponsive_time_secs = WMI_10X_VDEV_PARAM_AP_KEEPALIVE_MAX_UNRESPONSIVE_TIME_SECS, .ap_enable_nawds = WMI_10X_VDEV_PARAM_AP_ENABLE_NAWDS, .mcast2ucast_set = WMI_10X_VDEV_PARAM_MCAST2UCAST_SET, .enable_rtscts = WMI_10X_VDEV_PARAM_ENABLE_RTSCTS, .txbf = WMI_VDEV_PARAM_UNSUPPORTED, .packet_powersave = WMI_VDEV_PARAM_UNSUPPORTED, .drop_unencry = WMI_VDEV_PARAM_UNSUPPORTED, .tx_encap_type = WMI_VDEV_PARAM_UNSUPPORTED, .ap_detect_out_of_sync_sleeping_sta_time_secs = WMI_10X_VDEV_PARAM_AP_DETECT_OUT_OF_SYNC_SLEEPING_STA_TIME_SECS, .rc_num_retries = WMI_VDEV_PARAM_UNSUPPORTED, .cabq_maxdur = WMI_VDEV_PARAM_UNSUPPORTED, .mfptest_set = WMI_VDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_VDEV_PARAM_UNSUPPORTED, .vht_sgimask = WMI_VDEV_PARAM_UNSUPPORTED, .vht80_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_enable = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_tgt_bmiss_num = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_bmiss_sample_cycle = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_slop_step = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_init_slop = WMI_VDEV_PARAM_UNSUPPORTED, .early_rx_adjust_pause = WMI_VDEV_PARAM_UNSUPPORTED, .proxy_sta = WMI_VDEV_PARAM_UNSUPPORTED, .meru_vc = WMI_VDEV_PARAM_UNSUPPORTED, .rx_decap_type = WMI_VDEV_PARAM_UNSUPPORTED, .bw_nss_ratemask = WMI_VDEV_PARAM_UNSUPPORTED, .disable_4addr_src_lrn = WMI_VDEV_PARAM_UNSUPPORTED, .rtt_responder_role = WMI_VDEV_PARAM_UNSUPPORTED, }; static struct wmi_vdev_param_map wmi_10_4_vdev_param_map = { .rts_threshold = WMI_10_4_VDEV_PARAM_RTS_THRESHOLD, .fragmentation_threshold = WMI_10_4_VDEV_PARAM_FRAGMENTATION_THRESHOLD, .beacon_interval = WMI_10_4_VDEV_PARAM_BEACON_INTERVAL, .listen_interval = WMI_10_4_VDEV_PARAM_LISTEN_INTERVAL, .multicast_rate = WMI_10_4_VDEV_PARAM_MULTICAST_RATE, .mgmt_tx_rate = WMI_10_4_VDEV_PARAM_MGMT_TX_RATE, .slot_time = WMI_10_4_VDEV_PARAM_SLOT_TIME, .preamble = WMI_10_4_VDEV_PARAM_PREAMBLE, .swba_time = WMI_10_4_VDEV_PARAM_SWBA_TIME, .wmi_vdev_stats_update_period = WMI_10_4_VDEV_STATS_UPDATE_PERIOD, .wmi_vdev_pwrsave_ageout_time = WMI_10_4_VDEV_PWRSAVE_AGEOUT_TIME, .wmi_vdev_host_swba_interval = WMI_10_4_VDEV_HOST_SWBA_INTERVAL, .dtim_period = WMI_10_4_VDEV_PARAM_DTIM_PERIOD, .wmi_vdev_oc_scheduler_air_time_limit = WMI_10_4_VDEV_OC_SCHEDULER_AIR_TIME_LIMIT, .wds = WMI_10_4_VDEV_PARAM_WDS, .atim_window = WMI_10_4_VDEV_PARAM_ATIM_WINDOW, .bmiss_count_max = WMI_10_4_VDEV_PARAM_BMISS_COUNT_MAX, .bmiss_first_bcnt = WMI_10_4_VDEV_PARAM_BMISS_FIRST_BCNT, .bmiss_final_bcnt = WMI_10_4_VDEV_PARAM_BMISS_FINAL_BCNT, .feature_wmm = WMI_10_4_VDEV_PARAM_FEATURE_WMM, .chwidth = WMI_10_4_VDEV_PARAM_CHWIDTH, .chextoffset = WMI_10_4_VDEV_PARAM_CHEXTOFFSET, .disable_htprotection = WMI_10_4_VDEV_PARAM_DISABLE_HTPROTECTION, .sta_quickkickout = WMI_10_4_VDEV_PARAM_STA_QUICKKICKOUT, .mgmt_rate = WMI_10_4_VDEV_PARAM_MGMT_RATE, .protection_mode = WMI_10_4_VDEV_PARAM_PROTECTION_MODE, .fixed_rate = WMI_10_4_VDEV_PARAM_FIXED_RATE, .sgi = WMI_10_4_VDEV_PARAM_SGI, .ldpc = WMI_10_4_VDEV_PARAM_LDPC, .tx_stbc = WMI_10_4_VDEV_PARAM_TX_STBC, .rx_stbc = WMI_10_4_VDEV_PARAM_RX_STBC, .intra_bss_fwd = WMI_10_4_VDEV_PARAM_INTRA_BSS_FWD, .def_keyid = WMI_10_4_VDEV_PARAM_DEF_KEYID, .nss = WMI_10_4_VDEV_PARAM_NSS, .bcast_data_rate = WMI_10_4_VDEV_PARAM_BCAST_DATA_RATE, .mcast_data_rate = WMI_10_4_VDEV_PARAM_MCAST_DATA_RATE, .mcast_indicate = WMI_10_4_VDEV_PARAM_MCAST_INDICATE, .dhcp_indicate = WMI_10_4_VDEV_PARAM_DHCP_INDICATE, .unknown_dest_indicate = WMI_10_4_VDEV_PARAM_UNKNOWN_DEST_INDICATE, .ap_keepalive_min_idle_inactive_time_secs = WMI_10_4_VDEV_PARAM_AP_KEEPALIVE_MIN_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_idle_inactive_time_secs = WMI_10_4_VDEV_PARAM_AP_KEEPALIVE_MAX_IDLE_INACTIVE_TIME_SECS, .ap_keepalive_max_unresponsive_time_secs = WMI_10_4_VDEV_PARAM_AP_KEEPALIVE_MAX_UNRESPONSIVE_TIME_SECS, .ap_enable_nawds = WMI_10_4_VDEV_PARAM_AP_ENABLE_NAWDS, .mcast2ucast_set = WMI_10_4_VDEV_PARAM_MCAST2UCAST_SET, .enable_rtscts = WMI_10_4_VDEV_PARAM_ENABLE_RTSCTS, .txbf = WMI_10_4_VDEV_PARAM_TXBF, .packet_powersave = WMI_10_4_VDEV_PARAM_PACKET_POWERSAVE, .drop_unencry = WMI_10_4_VDEV_PARAM_DROP_UNENCRY, .tx_encap_type = WMI_10_4_VDEV_PARAM_TX_ENCAP_TYPE, .ap_detect_out_of_sync_sleeping_sta_time_secs = WMI_10_4_VDEV_PARAM_AP_DETECT_OUT_OF_SYNC_SLEEPING_STA_TIME_SECS, .rc_num_retries = WMI_10_4_VDEV_PARAM_RC_NUM_RETRIES, .cabq_maxdur = WMI_10_4_VDEV_PARAM_CABQ_MAXDUR, .mfptest_set = WMI_10_4_VDEV_PARAM_MFPTEST_SET, .rts_fixed_rate = WMI_10_4_VDEV_PARAM_RTS_FIXED_RATE, .vht_sgimask = WMI_10_4_VDEV_PARAM_VHT_SGIMASK, .vht80_ratemask = WMI_10_4_VDEV_PARAM_VHT80_RATEMASK, .early_rx_adjust_enable = WMI_10_4_VDEV_PARAM_EARLY_RX_ADJUST_ENABLE, .early_rx_tgt_bmiss_num = WMI_10_4_VDEV_PARAM_EARLY_RX_TGT_BMISS_NUM, .early_rx_bmiss_sample_cycle = WMI_10_4_VDEV_PARAM_EARLY_RX_BMISS_SAMPLE_CYCLE, .early_rx_slop_step = WMI_10_4_VDEV_PARAM_EARLY_RX_SLOP_STEP, .early_rx_init_slop = WMI_10_4_VDEV_PARAM_EARLY_RX_INIT_SLOP, .early_rx_adjust_pause = WMI_10_4_VDEV_PARAM_EARLY_RX_ADJUST_PAUSE, .proxy_sta = WMI_10_4_VDEV_PARAM_PROXY_STA, .meru_vc = WMI_10_4_VDEV_PARAM_MERU_VC, .rx_decap_type = WMI_10_4_VDEV_PARAM_RX_DECAP_TYPE, .bw_nss_ratemask = WMI_10_4_VDEV_PARAM_BW_NSS_RATEMASK, .inc_tsf = WMI_10_4_VDEV_PARAM_TSF_INCREMENT, .dec_tsf = WMI_10_4_VDEV_PARAM_TSF_DECREMENT, .disable_4addr_src_lrn = WMI_10_4_VDEV_PARAM_DISABLE_4_ADDR_SRC_LRN, .rtt_responder_role = WMI_10_4_VDEV_PARAM_ENABLE_DISABLE_RTT_RESPONDER_ROLE, }; static struct wmi_pdev_param_map wmi_pdev_param_map = { .tx_chain_mask = WMI_PDEV_PARAM_TX_CHAIN_MASK, .rx_chain_mask = WMI_PDEV_PARAM_RX_CHAIN_MASK, .txpower_limit2g = WMI_PDEV_PARAM_TXPOWER_LIMIT2G, .txpower_limit5g = WMI_PDEV_PARAM_TXPOWER_LIMIT5G, .txpower_scale = WMI_PDEV_PARAM_TXPOWER_SCALE, .beacon_gen_mode = WMI_PDEV_PARAM_BEACON_GEN_MODE, .beacon_tx_mode = WMI_PDEV_PARAM_BEACON_TX_MODE, .resmgr_offchan_mode = WMI_PDEV_PARAM_RESMGR_OFFCHAN_MODE, .protection_mode = WMI_PDEV_PARAM_PROTECTION_MODE, .dynamic_bw = WMI_PDEV_PARAM_DYNAMIC_BW, .non_agg_sw_retry_th = WMI_PDEV_PARAM_NON_AGG_SW_RETRY_TH, .agg_sw_retry_th = WMI_PDEV_PARAM_AGG_SW_RETRY_TH, .sta_kickout_th = WMI_PDEV_PARAM_STA_KICKOUT_TH, .ac_aggrsize_scaling = WMI_PDEV_PARAM_AC_AGGRSIZE_SCALING, .ltr_enable = WMI_PDEV_PARAM_LTR_ENABLE, .ltr_ac_latency_be = WMI_PDEV_PARAM_LTR_AC_LATENCY_BE, .ltr_ac_latency_bk = WMI_PDEV_PARAM_LTR_AC_LATENCY_BK, .ltr_ac_latency_vi = WMI_PDEV_PARAM_LTR_AC_LATENCY_VI, .ltr_ac_latency_vo = WMI_PDEV_PARAM_LTR_AC_LATENCY_VO, .ltr_ac_latency_timeout = WMI_PDEV_PARAM_LTR_AC_LATENCY_TIMEOUT, .ltr_sleep_override = WMI_PDEV_PARAM_LTR_SLEEP_OVERRIDE, .ltr_rx_override = WMI_PDEV_PARAM_LTR_RX_OVERRIDE, .ltr_tx_activity_timeout = WMI_PDEV_PARAM_LTR_TX_ACTIVITY_TIMEOUT, .l1ss_enable = WMI_PDEV_PARAM_L1SS_ENABLE, .dsleep_enable = WMI_PDEV_PARAM_DSLEEP_ENABLE, .pcielp_txbuf_flush = WMI_PDEV_PARAM_PCIELP_TXBUF_FLUSH, .pcielp_txbuf_watermark = WMI_PDEV_PARAM_PCIELP_TXBUF_TMO_EN, .pcielp_txbuf_tmo_en = WMI_PDEV_PARAM_PCIELP_TXBUF_TMO_EN, .pcielp_txbuf_tmo_value = WMI_PDEV_PARAM_PCIELP_TXBUF_TMO_VALUE, .pdev_stats_update_period = WMI_PDEV_PARAM_PDEV_STATS_UPDATE_PERIOD, .vdev_stats_update_period = WMI_PDEV_PARAM_VDEV_STATS_UPDATE_PERIOD, .peer_stats_update_period = WMI_PDEV_PARAM_PEER_STATS_UPDATE_PERIOD, .bcnflt_stats_update_period = WMI_PDEV_PARAM_BCNFLT_STATS_UPDATE_PERIOD, .pmf_qos = WMI_PDEV_PARAM_PMF_QOS, .arp_ac_override = WMI_PDEV_PARAM_ARP_AC_OVERRIDE, .dcs = WMI_PDEV_PARAM_DCS, .ani_enable = WMI_PDEV_PARAM_ANI_ENABLE, .ani_poll_period = WMI_PDEV_PARAM_ANI_POLL_PERIOD, .ani_listen_period = WMI_PDEV_PARAM_ANI_LISTEN_PERIOD, .ani_ofdm_level = WMI_PDEV_PARAM_ANI_OFDM_LEVEL, .ani_cck_level = WMI_PDEV_PARAM_ANI_CCK_LEVEL, .dyntxchain = WMI_PDEV_PARAM_DYNTXCHAIN, .proxy_sta = WMI_PDEV_PARAM_PROXY_STA, .idle_ps_config = WMI_PDEV_PARAM_IDLE_PS_CONFIG, .power_gating_sleep = WMI_PDEV_PARAM_POWER_GATING_SLEEP, .fast_channel_reset = WMI_PDEV_PARAM_UNSUPPORTED, .burst_dur = WMI_PDEV_PARAM_UNSUPPORTED, .burst_enable = WMI_PDEV_PARAM_UNSUPPORTED, .cal_period = WMI_PDEV_PARAM_UNSUPPORTED, .aggr_burst = WMI_PDEV_PARAM_UNSUPPORTED, .rx_decap_mode = WMI_PDEV_PARAM_UNSUPPORTED, .smart_antenna_default_antenna = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_override = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_tid = WMI_PDEV_PARAM_UNSUPPORTED, .antenna_gain = WMI_PDEV_PARAM_UNSUPPORTED, .rx_filter = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_to_ucast_tid = WMI_PDEV_PARAM_UNSUPPORTED, .proxy_sta_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .remove_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .peer_sta_ps_statechg_enable = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_ac_override = WMI_PDEV_PARAM_UNSUPPORTED, .block_interbss = WMI_PDEV_PARAM_UNSUPPORTED, .set_disable_reset_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_msdu_ttl_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_ppdu_duration_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .txbf_sound_period_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_promisc_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_burst_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .en_stats = WMI_PDEV_PARAM_UNSUPPORTED, .mu_group_policy = WMI_PDEV_PARAM_UNSUPPORTED, .noise_detection = WMI_PDEV_PARAM_UNSUPPORTED, .noise_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .dpd_enable = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_bcast_echo = WMI_PDEV_PARAM_UNSUPPORTED, .atf_strict_sch = WMI_PDEV_PARAM_UNSUPPORTED, .atf_sched_duration = WMI_PDEV_PARAM_UNSUPPORTED, .ant_plzn = WMI_PDEV_PARAM_UNSUPPORTED, .mgmt_retry_limit = WMI_PDEV_PARAM_UNSUPPORTED, .sensitivity_level = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_2g = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_5g = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_amsdu = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_ampdu = WMI_PDEV_PARAM_UNSUPPORTED, .cca_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_PDEV_PARAM_UNSUPPORTED, .pdev_reset = WMI_PDEV_PARAM_UNSUPPORTED, .wapi_mbssid_offset = WMI_PDEV_PARAM_UNSUPPORTED, .arp_srcaddr = WMI_PDEV_PARAM_UNSUPPORTED, .arp_dstaddr = WMI_PDEV_PARAM_UNSUPPORTED, .enable_btcoex = WMI_PDEV_PARAM_UNSUPPORTED, }; static struct wmi_pdev_param_map wmi_10x_pdev_param_map = { .tx_chain_mask = WMI_10X_PDEV_PARAM_TX_CHAIN_MASK, .rx_chain_mask = WMI_10X_PDEV_PARAM_RX_CHAIN_MASK, .txpower_limit2g = WMI_10X_PDEV_PARAM_TXPOWER_LIMIT2G, .txpower_limit5g = WMI_10X_PDEV_PARAM_TXPOWER_LIMIT5G, .txpower_scale = WMI_10X_PDEV_PARAM_TXPOWER_SCALE, .beacon_gen_mode = WMI_10X_PDEV_PARAM_BEACON_GEN_MODE, .beacon_tx_mode = WMI_10X_PDEV_PARAM_BEACON_TX_MODE, .resmgr_offchan_mode = WMI_10X_PDEV_PARAM_RESMGR_OFFCHAN_MODE, .protection_mode = WMI_10X_PDEV_PARAM_PROTECTION_MODE, .dynamic_bw = WMI_10X_PDEV_PARAM_DYNAMIC_BW, .non_agg_sw_retry_th = WMI_10X_PDEV_PARAM_NON_AGG_SW_RETRY_TH, .agg_sw_retry_th = WMI_10X_PDEV_PARAM_AGG_SW_RETRY_TH, .sta_kickout_th = WMI_10X_PDEV_PARAM_STA_KICKOUT_TH, .ac_aggrsize_scaling = WMI_10X_PDEV_PARAM_AC_AGGRSIZE_SCALING, .ltr_enable = WMI_10X_PDEV_PARAM_LTR_ENABLE, .ltr_ac_latency_be = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_BE, .ltr_ac_latency_bk = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_BK, .ltr_ac_latency_vi = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_VI, .ltr_ac_latency_vo = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_VO, .ltr_ac_latency_timeout = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_TIMEOUT, .ltr_sleep_override = WMI_10X_PDEV_PARAM_LTR_SLEEP_OVERRIDE, .ltr_rx_override = WMI_10X_PDEV_PARAM_LTR_RX_OVERRIDE, .ltr_tx_activity_timeout = WMI_10X_PDEV_PARAM_LTR_TX_ACTIVITY_TIMEOUT, .l1ss_enable = WMI_10X_PDEV_PARAM_L1SS_ENABLE, .dsleep_enable = WMI_10X_PDEV_PARAM_DSLEEP_ENABLE, .pcielp_txbuf_flush = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_watermark = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_tmo_en = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_tmo_value = WMI_PDEV_PARAM_UNSUPPORTED, .pdev_stats_update_period = WMI_10X_PDEV_PARAM_PDEV_STATS_UPDATE_PERIOD, .vdev_stats_update_period = WMI_10X_PDEV_PARAM_VDEV_STATS_UPDATE_PERIOD, .peer_stats_update_period = WMI_10X_PDEV_PARAM_PEER_STATS_UPDATE_PERIOD, .bcnflt_stats_update_period = WMI_10X_PDEV_PARAM_BCNFLT_STATS_UPDATE_PERIOD, .pmf_qos = WMI_10X_PDEV_PARAM_PMF_QOS, .arp_ac_override = WMI_10X_PDEV_PARAM_ARPDHCP_AC_OVERRIDE, .dcs = WMI_10X_PDEV_PARAM_DCS, .ani_enable = WMI_10X_PDEV_PARAM_ANI_ENABLE, .ani_poll_period = WMI_10X_PDEV_PARAM_ANI_POLL_PERIOD, .ani_listen_period = WMI_10X_PDEV_PARAM_ANI_LISTEN_PERIOD, .ani_ofdm_level = WMI_10X_PDEV_PARAM_ANI_OFDM_LEVEL, .ani_cck_level = WMI_10X_PDEV_PARAM_ANI_CCK_LEVEL, .dyntxchain = WMI_10X_PDEV_PARAM_DYNTXCHAIN, .proxy_sta = WMI_PDEV_PARAM_UNSUPPORTED, .idle_ps_config = WMI_PDEV_PARAM_UNSUPPORTED, .power_gating_sleep = WMI_PDEV_PARAM_UNSUPPORTED, .fast_channel_reset = WMI_10X_PDEV_PARAM_FAST_CHANNEL_RESET, .burst_dur = WMI_10X_PDEV_PARAM_BURST_DUR, .burst_enable = WMI_10X_PDEV_PARAM_BURST_ENABLE, .cal_period = WMI_10X_PDEV_PARAM_CAL_PERIOD, .aggr_burst = WMI_PDEV_PARAM_UNSUPPORTED, .rx_decap_mode = WMI_PDEV_PARAM_UNSUPPORTED, .smart_antenna_default_antenna = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_override = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_tid = WMI_PDEV_PARAM_UNSUPPORTED, .antenna_gain = WMI_PDEV_PARAM_UNSUPPORTED, .rx_filter = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_to_ucast_tid = WMI_PDEV_PARAM_UNSUPPORTED, .proxy_sta_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .remove_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .peer_sta_ps_statechg_enable = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_ac_override = WMI_PDEV_PARAM_UNSUPPORTED, .block_interbss = WMI_PDEV_PARAM_UNSUPPORTED, .set_disable_reset_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_msdu_ttl_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_ppdu_duration_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .txbf_sound_period_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_promisc_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_burst_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .en_stats = WMI_PDEV_PARAM_UNSUPPORTED, .mu_group_policy = WMI_PDEV_PARAM_UNSUPPORTED, .noise_detection = WMI_PDEV_PARAM_UNSUPPORTED, .noise_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .dpd_enable = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_bcast_echo = WMI_PDEV_PARAM_UNSUPPORTED, .atf_strict_sch = WMI_PDEV_PARAM_UNSUPPORTED, .atf_sched_duration = WMI_PDEV_PARAM_UNSUPPORTED, .ant_plzn = WMI_PDEV_PARAM_UNSUPPORTED, .mgmt_retry_limit = WMI_PDEV_PARAM_UNSUPPORTED, .sensitivity_level = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_2g = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_5g = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_amsdu = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_ampdu = WMI_PDEV_PARAM_UNSUPPORTED, .cca_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_PDEV_PARAM_UNSUPPORTED, .pdev_reset = WMI_PDEV_PARAM_UNSUPPORTED, .wapi_mbssid_offset = WMI_PDEV_PARAM_UNSUPPORTED, .arp_srcaddr = WMI_PDEV_PARAM_UNSUPPORTED, .arp_dstaddr = WMI_PDEV_PARAM_UNSUPPORTED, .enable_btcoex = WMI_PDEV_PARAM_UNSUPPORTED, }; static struct wmi_pdev_param_map wmi_10_2_4_pdev_param_map = { .tx_chain_mask = WMI_10X_PDEV_PARAM_TX_CHAIN_MASK, .rx_chain_mask = WMI_10X_PDEV_PARAM_RX_CHAIN_MASK, .txpower_limit2g = WMI_10X_PDEV_PARAM_TXPOWER_LIMIT2G, .txpower_limit5g = WMI_10X_PDEV_PARAM_TXPOWER_LIMIT5G, .txpower_scale = WMI_10X_PDEV_PARAM_TXPOWER_SCALE, .beacon_gen_mode = WMI_10X_PDEV_PARAM_BEACON_GEN_MODE, .beacon_tx_mode = WMI_10X_PDEV_PARAM_BEACON_TX_MODE, .resmgr_offchan_mode = WMI_10X_PDEV_PARAM_RESMGR_OFFCHAN_MODE, .protection_mode = WMI_10X_PDEV_PARAM_PROTECTION_MODE, .dynamic_bw = WMI_10X_PDEV_PARAM_DYNAMIC_BW, .non_agg_sw_retry_th = WMI_10X_PDEV_PARAM_NON_AGG_SW_RETRY_TH, .agg_sw_retry_th = WMI_10X_PDEV_PARAM_AGG_SW_RETRY_TH, .sta_kickout_th = WMI_10X_PDEV_PARAM_STA_KICKOUT_TH, .ac_aggrsize_scaling = WMI_10X_PDEV_PARAM_AC_AGGRSIZE_SCALING, .ltr_enable = WMI_10X_PDEV_PARAM_LTR_ENABLE, .ltr_ac_latency_be = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_BE, .ltr_ac_latency_bk = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_BK, .ltr_ac_latency_vi = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_VI, .ltr_ac_latency_vo = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_VO, .ltr_ac_latency_timeout = WMI_10X_PDEV_PARAM_LTR_AC_LATENCY_TIMEOUT, .ltr_sleep_override = WMI_10X_PDEV_PARAM_LTR_SLEEP_OVERRIDE, .ltr_rx_override = WMI_10X_PDEV_PARAM_LTR_RX_OVERRIDE, .ltr_tx_activity_timeout = WMI_10X_PDEV_PARAM_LTR_TX_ACTIVITY_TIMEOUT, .l1ss_enable = WMI_10X_PDEV_PARAM_L1SS_ENABLE, .dsleep_enable = WMI_10X_PDEV_PARAM_DSLEEP_ENABLE, .pcielp_txbuf_flush = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_watermark = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_tmo_en = WMI_PDEV_PARAM_UNSUPPORTED, .pcielp_txbuf_tmo_value = WMI_PDEV_PARAM_UNSUPPORTED, .pdev_stats_update_period = WMI_10X_PDEV_PARAM_PDEV_STATS_UPDATE_PERIOD, .vdev_stats_update_period = WMI_10X_PDEV_PARAM_VDEV_STATS_UPDATE_PERIOD, .peer_stats_update_period = WMI_10X_PDEV_PARAM_PEER_STATS_UPDATE_PERIOD, .bcnflt_stats_update_period = WMI_10X_PDEV_PARAM_BCNFLT_STATS_UPDATE_PERIOD, .pmf_qos = WMI_10X_PDEV_PARAM_PMF_QOS, .arp_ac_override = WMI_10X_PDEV_PARAM_ARPDHCP_AC_OVERRIDE, .dcs = WMI_10X_PDEV_PARAM_DCS, .ani_enable = WMI_10X_PDEV_PARAM_ANI_ENABLE, .ani_poll_period = WMI_10X_PDEV_PARAM_ANI_POLL_PERIOD, .ani_listen_period = WMI_10X_PDEV_PARAM_ANI_LISTEN_PERIOD, .ani_ofdm_level = WMI_10X_PDEV_PARAM_ANI_OFDM_LEVEL, .ani_cck_level = WMI_10X_PDEV_PARAM_ANI_CCK_LEVEL, .dyntxchain = WMI_10X_PDEV_PARAM_DYNTXCHAIN, .proxy_sta = WMI_PDEV_PARAM_UNSUPPORTED, .idle_ps_config = WMI_PDEV_PARAM_UNSUPPORTED, .power_gating_sleep = WMI_PDEV_PARAM_UNSUPPORTED, .fast_channel_reset = WMI_10X_PDEV_PARAM_FAST_CHANNEL_RESET, .burst_dur = WMI_10X_PDEV_PARAM_BURST_DUR, .burst_enable = WMI_10X_PDEV_PARAM_BURST_ENABLE, .cal_period = WMI_10X_PDEV_PARAM_CAL_PERIOD, .aggr_burst = WMI_PDEV_PARAM_UNSUPPORTED, .rx_decap_mode = WMI_PDEV_PARAM_UNSUPPORTED, .smart_antenna_default_antenna = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_override = WMI_PDEV_PARAM_UNSUPPORTED, .igmpmld_tid = WMI_PDEV_PARAM_UNSUPPORTED, .antenna_gain = WMI_PDEV_PARAM_UNSUPPORTED, .rx_filter = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_to_ucast_tid = WMI_PDEV_PARAM_UNSUPPORTED, .proxy_sta_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_mode = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .remove_mcast2ucast_buffer = WMI_PDEV_PARAM_UNSUPPORTED, .peer_sta_ps_statechg_enable = WMI_10X_PDEV_PARAM_PEER_STA_PS_STATECHG_ENABLE, .igmpmld_ac_override = WMI_PDEV_PARAM_UNSUPPORTED, .block_interbss = WMI_PDEV_PARAM_UNSUPPORTED, .set_disable_reset_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_msdu_ttl_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_ppdu_duration_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .txbf_sound_period_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_promisc_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .set_burst_mode_cmdid = WMI_PDEV_PARAM_UNSUPPORTED, .en_stats = WMI_PDEV_PARAM_UNSUPPORTED, .mu_group_policy = WMI_PDEV_PARAM_UNSUPPORTED, .noise_detection = WMI_PDEV_PARAM_UNSUPPORTED, .noise_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .dpd_enable = WMI_PDEV_PARAM_UNSUPPORTED, .set_mcast_bcast_echo = WMI_PDEV_PARAM_UNSUPPORTED, .atf_strict_sch = WMI_PDEV_PARAM_UNSUPPORTED, .atf_sched_duration = WMI_PDEV_PARAM_UNSUPPORTED, .ant_plzn = WMI_PDEV_PARAM_UNSUPPORTED, .mgmt_retry_limit = WMI_PDEV_PARAM_UNSUPPORTED, .sensitivity_level = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_2g = WMI_PDEV_PARAM_UNSUPPORTED, .signed_txpower_5g = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_amsdu = WMI_PDEV_PARAM_UNSUPPORTED, .enable_per_tid_ampdu = WMI_PDEV_PARAM_UNSUPPORTED, .cca_threshold = WMI_PDEV_PARAM_UNSUPPORTED, .rts_fixed_rate = WMI_PDEV_PARAM_UNSUPPORTED, .pdev_reset = WMI_10X_PDEV_PARAM_PDEV_RESET, .wapi_mbssid_offset = WMI_PDEV_PARAM_UNSUPPORTED, .arp_srcaddr = WMI_PDEV_PARAM_UNSUPPORTED, .arp_dstaddr = WMI_PDEV_PARAM_UNSUPPORTED, .enable_btcoex = WMI_PDEV_PARAM_UNSUPPORTED, }; /* firmware 10.2 specific mappings */ static struct wmi_cmd_map wmi_10_2_cmd_map = { .init_cmdid = WMI_10_2_INIT_CMDID, .start_scan_cmdid = WMI_10_2_START_SCAN_CMDID, .stop_scan_cmdid = WMI_10_2_STOP_SCAN_CMDID, .scan_chan_list_cmdid = WMI_10_2_SCAN_CHAN_LIST_CMDID, .scan_sch_prio_tbl_cmdid = WMI_CMD_UNSUPPORTED, .scan_prob_req_oui_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_regdomain_cmdid = WMI_10_2_PDEV_SET_REGDOMAIN_CMDID, .pdev_set_channel_cmdid = WMI_10_2_PDEV_SET_CHANNEL_CMDID, .pdev_set_param_cmdid = WMI_10_2_PDEV_SET_PARAM_CMDID, .pdev_pktlog_enable_cmdid = WMI_10_2_PDEV_PKTLOG_ENABLE_CMDID, .pdev_pktlog_disable_cmdid = WMI_10_2_PDEV_PKTLOG_DISABLE_CMDID, .pdev_set_wmm_params_cmdid = WMI_10_2_PDEV_SET_WMM_PARAMS_CMDID, .pdev_set_ht_cap_ie_cmdid = WMI_10_2_PDEV_SET_HT_CAP_IE_CMDID, .pdev_set_vht_cap_ie_cmdid = WMI_10_2_PDEV_SET_VHT_CAP_IE_CMDID, .pdev_set_quiet_mode_cmdid = WMI_10_2_PDEV_SET_QUIET_MODE_CMDID, .pdev_green_ap_ps_enable_cmdid = WMI_10_2_PDEV_GREEN_AP_PS_ENABLE_CMDID, .pdev_get_tpc_config_cmdid = WMI_10_2_PDEV_GET_TPC_CONFIG_CMDID, .pdev_set_base_macaddr_cmdid = WMI_10_2_PDEV_SET_BASE_MACADDR_CMDID, .vdev_create_cmdid = WMI_10_2_VDEV_CREATE_CMDID, .vdev_delete_cmdid = WMI_10_2_VDEV_DELETE_CMDID, .vdev_start_request_cmdid = WMI_10_2_VDEV_START_REQUEST_CMDID, .vdev_restart_request_cmdid = WMI_10_2_VDEV_RESTART_REQUEST_CMDID, .vdev_up_cmdid = WMI_10_2_VDEV_UP_CMDID, .vdev_stop_cmdid = WMI_10_2_VDEV_STOP_CMDID, .vdev_down_cmdid = WMI_10_2_VDEV_DOWN_CMDID, .vdev_set_param_cmdid = WMI_10_2_VDEV_SET_PARAM_CMDID, .vdev_install_key_cmdid = WMI_10_2_VDEV_INSTALL_KEY_CMDID, .peer_create_cmdid = WMI_10_2_PEER_CREATE_CMDID, .peer_delete_cmdid = WMI_10_2_PEER_DELETE_CMDID, .peer_flush_tids_cmdid = WMI_10_2_PEER_FLUSH_TIDS_CMDID, .peer_set_param_cmdid = WMI_10_2_PEER_SET_PARAM_CMDID, .peer_assoc_cmdid = WMI_10_2_PEER_ASSOC_CMDID, .peer_add_wds_entry_cmdid = WMI_10_2_PEER_ADD_WDS_ENTRY_CMDID, .peer_remove_wds_entry_cmdid = WMI_10_2_PEER_REMOVE_WDS_ENTRY_CMDID, .peer_mcast_group_cmdid = WMI_10_2_PEER_MCAST_GROUP_CMDID, .bcn_tx_cmdid = WMI_10_2_BCN_TX_CMDID, .pdev_send_bcn_cmdid = WMI_10_2_PDEV_SEND_BCN_CMDID, .bcn_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .bcn_filter_rx_cmdid = WMI_10_2_BCN_FILTER_RX_CMDID, .prb_req_filter_rx_cmdid = WMI_10_2_PRB_REQ_FILTER_RX_CMDID, .mgmt_tx_cmdid = WMI_10_2_MGMT_TX_CMDID, .prb_tmpl_cmdid = WMI_CMD_UNSUPPORTED, .addba_clear_resp_cmdid = WMI_10_2_ADDBA_CLEAR_RESP_CMDID, .addba_send_cmdid = WMI_10_2_ADDBA_SEND_CMDID, .addba_status_cmdid = WMI_10_2_ADDBA_STATUS_CMDID, .delba_send_cmdid = WMI_10_2_DELBA_SEND_CMDID, .addba_set_resp_cmdid = WMI_10_2_ADDBA_SET_RESP_CMDID, .send_singleamsdu_cmdid = WMI_10_2_SEND_SINGLEAMSDU_CMDID, .sta_powersave_mode_cmdid = WMI_10_2_STA_POWERSAVE_MODE_CMDID, .sta_powersave_param_cmdid = WMI_10_2_STA_POWERSAVE_PARAM_CMDID, .sta_mimo_ps_mode_cmdid = WMI_10_2_STA_MIMO_PS_MODE_CMDID, .pdev_dfs_enable_cmdid = WMI_10_2_PDEV_DFS_ENABLE_CMDID, .pdev_dfs_disable_cmdid = WMI_10_2_PDEV_DFS_DISABLE_CMDID, .roam_scan_mode = WMI_10_2_ROAM_SCAN_MODE, .roam_scan_rssi_threshold = WMI_10_2_ROAM_SCAN_RSSI_THRESHOLD, .roam_scan_period = WMI_10_2_ROAM_SCAN_PERIOD, .roam_scan_rssi_change_threshold = WMI_10_2_ROAM_SCAN_RSSI_CHANGE_THRESHOLD, .roam_ap_profile = WMI_10_2_ROAM_AP_PROFILE, .ofl_scan_add_ap_profile = WMI_10_2_OFL_SCAN_ADD_AP_PROFILE, .ofl_scan_remove_ap_profile = WMI_10_2_OFL_SCAN_REMOVE_AP_PROFILE, .ofl_scan_period = WMI_10_2_OFL_SCAN_PERIOD, .p2p_dev_set_device_info = WMI_10_2_P2P_DEV_SET_DEVICE_INFO, .p2p_dev_set_discoverability = WMI_10_2_P2P_DEV_SET_DISCOVERABILITY, .p2p_go_set_beacon_ie = WMI_10_2_P2P_GO_SET_BEACON_IE, .p2p_go_set_probe_resp_ie = WMI_10_2_P2P_GO_SET_PROBE_RESP_IE, .p2p_set_vendor_ie_data_cmdid = WMI_CMD_UNSUPPORTED, .ap_ps_peer_param_cmdid = WMI_10_2_AP_PS_PEER_PARAM_CMDID, .ap_ps_peer_uapsd_coex_cmdid = WMI_CMD_UNSUPPORTED, .peer_rate_retry_sched_cmdid = WMI_10_2_PEER_RATE_RETRY_SCHED_CMDID, .wlan_profile_trigger_cmdid = WMI_10_2_WLAN_PROFILE_TRIGGER_CMDID, .wlan_profile_set_hist_intvl_cmdid = WMI_10_2_WLAN_PROFILE_SET_HIST_INTVL_CMDID, .wlan_profile_get_profile_data_cmdid = WMI_10_2_WLAN_PROFILE_GET_PROFILE_DATA_CMDID, .wlan_profile_enable_profile_id_cmdid = WMI_10_2_WLAN_PROFILE_ENABLE_PROFILE_ID_CMDID, .wlan_profile_list_profile_id_cmdid = WMI_10_2_WLAN_PROFILE_LIST_PROFILE_ID_CMDID, .pdev_suspend_cmdid = WMI_10_2_PDEV_SUSPEND_CMDID, .pdev_resume_cmdid = WMI_10_2_PDEV_RESUME_CMDID, .add_bcn_filter_cmdid = WMI_10_2_ADD_BCN_FILTER_CMDID, .rmv_bcn_filter_cmdid = WMI_10_2_RMV_BCN_FILTER_CMDID, .wow_add_wake_pattern_cmdid = WMI_10_2_WOW_ADD_WAKE_PATTERN_CMDID, .wow_del_wake_pattern_cmdid = WMI_10_2_WOW_DEL_WAKE_PATTERN_CMDID, .wow_enable_disable_wake_event_cmdid = WMI_10_2_WOW_ENABLE_DISABLE_WAKE_EVENT_CMDID, .wow_enable_cmdid = WMI_10_2_WOW_ENABLE_CMDID, .wow_hostwakeup_from_sleep_cmdid = WMI_10_2_WOW_HOSTWAKEUP_FROM_SLEEP_CMDID, .rtt_measreq_cmdid = WMI_10_2_RTT_MEASREQ_CMDID, .rtt_tsf_cmdid = WMI_10_2_RTT_TSF_CMDID, .vdev_spectral_scan_configure_cmdid = WMI_10_2_VDEV_SPECTRAL_SCAN_CONFIGURE_CMDID, .vdev_spectral_scan_enable_cmdid = WMI_10_2_VDEV_SPECTRAL_SCAN_ENABLE_CMDID, .request_stats_cmdid = WMI_10_2_REQUEST_STATS_CMDID, .set_arp_ns_offload_cmdid = WMI_CMD_UNSUPPORTED, .network_list_offload_config_cmdid = WMI_CMD_UNSUPPORTED, .gtk_offload_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_enable_cmdid = WMI_CMD_UNSUPPORTED, .csa_offload_chanswitch_cmdid = WMI_CMD_UNSUPPORTED, .chatter_set_mode_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_addba_cmdid = WMI_CMD_UNSUPPORTED, .peer_tid_delba_cmdid = WMI_CMD_UNSUPPORTED, .sta_dtim_ps_method_cmdid = WMI_CMD_UNSUPPORTED, .sta_uapsd_auto_trig_cmdid = WMI_CMD_UNSUPPORTED, .sta_keepalive_cmd = WMI_CMD_UNSUPPORTED, .echo_cmdid = WMI_10_2_ECHO_CMDID, .pdev_utf_cmdid = WMI_10_2_PDEV_UTF_CMDID, .dbglog_cfg_cmdid = WMI_10_2_DBGLOG_CFG_CMDID, .pdev_qvit_cmdid = WMI_10_2_PDEV_QVIT_CMDID, .pdev_ftm_intg_cmdid = WMI_CMD_UNSUPPORTED, .vdev_set_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .vdev_get_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .force_fw_hang_cmdid = WMI_CMD_UNSUPPORTED, .gpio_config_cmdid = WMI_10_2_GPIO_CONFIG_CMDID, .gpio_output_cmdid = WMI_10_2_GPIO_OUTPUT_CMDID, .pdev_get_temperature_cmdid = WMI_CMD_UNSUPPORTED, .pdev_enable_adaptive_cca_cmdid = WMI_CMD_UNSUPPORTED, .scan_update_request_cmdid = WMI_CMD_UNSUPPORTED, .vdev_standby_response_cmdid = WMI_CMD_UNSUPPORTED, .vdev_resume_response_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_add_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_evict_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_restore_peer_cmdid = WMI_CMD_UNSUPPORTED, .wlan_peer_caching_print_all_peers_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_update_wds_entry_cmdid = WMI_CMD_UNSUPPORTED, .peer_add_proxy_sta_entry_cmdid = WMI_CMD_UNSUPPORTED, .rtt_keepalive_cmdid = WMI_CMD_UNSUPPORTED, .oem_req_cmdid = WMI_CMD_UNSUPPORTED, .nan_cmdid = WMI_CMD_UNSUPPORTED, .vdev_ratemask_cmdid = WMI_CMD_UNSUPPORTED, .qboost_cfg_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_enable_cmdid = WMI_CMD_UNSUPPORTED, .pdev_smart_ant_set_rx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_tx_antenna_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_train_info_cmdid = WMI_CMD_UNSUPPORTED, .peer_smart_ant_set_node_config_ops_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_antenna_switch_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_ctl_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_set_mimogain_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_ratepwr_chainmsk_table_cmdid = WMI_CMD_UNSUPPORTED, .pdev_fips_cmdid = WMI_CMD_UNSUPPORTED, .tt_set_conf_cmdid = WMI_CMD_UNSUPPORTED, .fwtest_cmdid = WMI_CMD_UNSUPPORTED, .vdev_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .peer_atf_request_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_cck_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_ani_ofdm_config_cmdid = WMI_CMD_UNSUPPORTED, .pdev_reserve_ast_entry_cmdid = WMI_CMD_UNSUPPORTED, .pdev_get_tpc_table_cmdid = WMI_CMD_UNSUPPORTED, .radar_found_cmdid = WMI_CMD_UNSUPPORTED, }; static struct wmi_pdev_param_map wmi_10_4_pdev_param_map = { .tx_chain_mask = WMI_10_4_PDEV_PARAM_TX_CHAIN_MASK, .rx_chain_mask = WMI_10_4_PDEV_PARAM_RX_CHAIN_MASK, .txpower_limit2g = WMI_10_4_PDEV_PARAM_TXPOWER_LIMIT2G, .txpower_limit5g = WMI_10_4_PDEV_PARAM_TXPOWER_LIMIT5G, .txpower_scale = WMI_10_4_PDEV_PARAM_TXPOWER_SCALE, .beacon_gen_mode = WMI_10_4_PDEV_PARAM_BEACON_GEN_MODE, .beacon_tx_mode = WMI_10_4_PDEV_PARAM_BEACON_TX_MODE, .resmgr_offchan_mode = WMI_10_4_PDEV_PARAM_RESMGR_OFFCHAN_MODE, .protection_mode = WMI_10_4_PDEV_PARAM_PROTECTION_MODE, .dynamic_bw = WMI_10_4_PDEV_PARAM_DYNAMIC_BW, .non_agg_sw_retry_th = WMI_10_4_PDEV_PARAM_NON_AGG_SW_RETRY_TH, .agg_sw_retry_th = WMI_10_4_PDEV_PARAM_AGG_SW_RETRY_TH, .sta_kickout_th = WMI_10_4_PDEV_PARAM_STA_KICKOUT_TH, .ac_aggrsize_scaling = WMI_10_4_PDEV_PARAM_AC_AGGRSIZE_SCALING, .ltr_enable = WMI_10_4_PDEV_PARAM_LTR_ENABLE, .ltr_ac_latency_be = WMI_10_4_PDEV_PARAM_LTR_AC_LATENCY_BE, .ltr_ac_latency_bk = WMI_10_4_PDEV_PARAM_LTR_AC_LATENCY_BK, .ltr_ac_latency_vi = WMI_10_4_PDEV_PARAM_LTR_AC_LATENCY_VI, .ltr_ac_latency_vo = WMI_10_4_PDEV_PARAM_LTR_AC_LATENCY_VO, .ltr_ac_latency_timeout = WMI_10_4_PDEV_PARAM_LTR_AC_LATENCY_TIMEOUT, .ltr_sleep_override = WMI_10_4_PDEV_PARAM_LTR_SLEEP_OVERRIDE, .ltr_rx_override = WMI_10_4_PDEV_PARAM_LTR_RX_OVERRIDE, .ltr_tx_activity_timeout = WMI_10_4_PDEV_PARAM_LTR_TX_ACTIVITY_TIMEOUT, .l1ss_enable = WMI_10_4_PDEV_PARAM_L1SS_ENABLE, .dsleep_enable = WMI_10_4_PDEV_PARAM_DSLEEP_ENABLE, .pcielp_txbuf_flush = WMI_10_4_PDEV_PARAM_PCIELP_TXBUF_FLUSH, .pcielp_txbuf_watermark = WMI_10_4_PDEV_PARAM_PCIELP_TXBUF_WATERMARK, .pcielp_txbuf_tmo_en = WMI_10_4_PDEV_PARAM_PCIELP_TXBUF_TMO_EN, .pcielp_txbuf_tmo_value = WMI_10_4_PDEV_PARAM_PCIELP_TXBUF_TMO_VALUE, .pdev_stats_update_period = WMI_10_4_PDEV_PARAM_PDEV_STATS_UPDATE_PERIOD, .vdev_stats_update_period = WMI_10_4_PDEV_PARAM_VDEV_STATS_UPDATE_PERIOD, .peer_stats_update_period = WMI_10_4_PDEV_PARAM_PEER_STATS_UPDATE_PERIOD, .bcnflt_stats_update_period = WMI_10_4_PDEV_PARAM_BCNFLT_STATS_UPDATE_PERIOD, .pmf_qos = WMI_10_4_PDEV_PARAM_PMF_QOS, .arp_ac_override = WMI_10_4_PDEV_PARAM_ARP_AC_OVERRIDE, .dcs = WMI_10_4_PDEV_PARAM_DCS, .ani_enable = WMI_10_4_PDEV_PARAM_ANI_ENABLE, .ani_poll_period = WMI_10_4_PDEV_PARAM_ANI_POLL_PERIOD, .ani_listen_period = WMI_10_4_PDEV_PARAM_ANI_LISTEN_PERIOD, .ani_ofdm_level = WMI_10_4_PDEV_PARAM_ANI_OFDM_LEVEL, .ani_cck_level = WMI_10_4_PDEV_PARAM_ANI_CCK_LEVEL, .dyntxchain = WMI_10_4_PDEV_PARAM_DYNTXCHAIN, .proxy_sta = WMI_10_4_PDEV_PARAM_PROXY_STA, .idle_ps_config = WMI_10_4_PDEV_PARAM_IDLE_PS_CONFIG, .power_gating_sleep = WMI_10_4_PDEV_PARAM_POWER_GATING_SLEEP, .fast_channel_reset = WMI_10_4_PDEV_PARAM_FAST_CHANNEL_RESET, .burst_dur = WMI_10_4_PDEV_PARAM_BURST_DUR, .burst_enable = WMI_10_4_PDEV_PARAM_BURST_ENABLE, .cal_period = WMI_10_4_PDEV_PARAM_CAL_PERIOD, .aggr_burst = WMI_10_4_PDEV_PARAM_AGGR_BURST, .rx_decap_mode = WMI_10_4_PDEV_PARAM_RX_DECAP_MODE, .smart_antenna_default_antenna = WMI_10_4_PDEV_PARAM_SMART_ANTENNA_DEFAULT_ANTENNA, .igmpmld_override = WMI_10_4_PDEV_PARAM_IGMPMLD_OVERRIDE, .igmpmld_tid = WMI_10_4_PDEV_PARAM_IGMPMLD_TID, .antenna_gain = WMI_10_4_PDEV_PARAM_ANTENNA_GAIN, .rx_filter = WMI_10_4_PDEV_PARAM_RX_FILTER, .set_mcast_to_ucast_tid = WMI_10_4_PDEV_SET_MCAST_TO_UCAST_TID, .proxy_sta_mode = WMI_10_4_PDEV_PARAM_PROXY_STA_MODE, .set_mcast2ucast_mode = WMI_10_4_PDEV_PARAM_SET_MCAST2UCAST_MODE, .set_mcast2ucast_buffer = WMI_10_4_PDEV_PARAM_SET_MCAST2UCAST_BUFFER, .remove_mcast2ucast_buffer = WMI_10_4_PDEV_PARAM_REMOVE_MCAST2UCAST_BUFFER, .peer_sta_ps_statechg_enable = WMI_10_4_PDEV_PEER_STA_PS_STATECHG_ENABLE, .igmpmld_ac_override = WMI_10_4_PDEV_PARAM_IGMPMLD_AC_OVERRIDE, .block_interbss = WMI_10_4_PDEV_PARAM_BLOCK_INTERBSS, .set_disable_reset_cmdid = WMI_10_4_PDEV_PARAM_SET_DISABLE_RESET_CMDID, .set_msdu_ttl_cmdid = WMI_10_4_PDEV_PARAM_SET_MSDU_TTL_CMDID, .set_ppdu_duration_cmdid = WMI_10_4_PDEV_PARAM_SET_PPDU_DURATION_CMDID, .txbf_sound_period_cmdid = WMI_10_4_PDEV_PARAM_TXBF_SOUND_PERIOD_CMDID, .set_promisc_mode_cmdid = WMI_10_4_PDEV_PARAM_SET_PROMISC_MODE_CMDID, .set_burst_mode_cmdid = WMI_10_4_PDEV_PARAM_SET_BURST_MODE_CMDID, .en_stats = WMI_10_4_PDEV_PARAM_EN_STATS, .mu_group_policy = WMI_10_4_PDEV_PARAM_MU_GROUP_POLICY, .noise_detection = WMI_10_4_PDEV_PARAM_NOISE_DETECTION, .noise_threshold = WMI_10_4_PDEV_PARAM_NOISE_THRESHOLD, .dpd_enable = WMI_10_4_PDEV_PARAM_DPD_ENABLE, .set_mcast_bcast_echo = WMI_10_4_PDEV_PARAM_SET_MCAST_BCAST_ECHO, .atf_strict_sch = WMI_10_4_PDEV_PARAM_ATF_STRICT_SCH, .atf_sched_duration = WMI_10_4_PDEV_PARAM_ATF_SCHED_DURATION, .ant_plzn = WMI_10_4_PDEV_PARAM_ANT_PLZN, .mgmt_retry_limit = WMI_10_4_PDEV_PARAM_MGMT_RETRY_LIMIT, .sensitivity_level = WMI_10_4_PDEV_PARAM_SENSITIVITY_LEVEL, .signed_txpower_2g = WMI_10_4_PDEV_PARAM_SIGNED_TXPOWER_2G, .signed_txpower_5g = WMI_10_4_PDEV_PARAM_SIGNED_TXPOWER_5G, .enable_per_tid_amsdu = WMI_10_4_PDEV_PARAM_ENABLE_PER_TID_AMSDU, .enable_per_tid_ampdu = WMI_10_4_PDEV_PARAM_ENABLE_PER_TID_AMPDU, .cca_threshold = WMI_10_4_PDEV_PARAM_CCA_THRESHOLD, .rts_fixed_rate = WMI_10_4_PDEV_PARAM_RTS_FIXED_RATE, .pdev_reset = WMI_10_4_PDEV_PARAM_PDEV_RESET, .wapi_mbssid_offset = WMI_10_4_PDEV_PARAM_WAPI_MBSSID_OFFSET, .arp_srcaddr = WMI_10_4_PDEV_PARAM_ARP_SRCADDR, .arp_dstaddr = WMI_10_4_PDEV_PARAM_ARP_DSTADDR, .enable_btcoex = WMI_10_4_PDEV_PARAM_ENABLE_BTCOEX, }; static const u8 wmi_key_cipher_suites[] = { [WMI_CIPHER_NONE] = WMI_CIPHER_NONE, [WMI_CIPHER_WEP] = WMI_CIPHER_WEP, [WMI_CIPHER_TKIP] = WMI_CIPHER_TKIP, [WMI_CIPHER_AES_OCB] = WMI_CIPHER_AES_OCB, [WMI_CIPHER_AES_CCM] = WMI_CIPHER_AES_CCM, [WMI_CIPHER_WAPI] = WMI_CIPHER_WAPI, [WMI_CIPHER_CKIP] = WMI_CIPHER_CKIP, [WMI_CIPHER_AES_CMAC] = WMI_CIPHER_AES_CMAC, [WMI_CIPHER_AES_GCM] = WMI_CIPHER_AES_GCM, }; static const u8 wmi_tlv_key_cipher_suites[] = { [WMI_CIPHER_NONE] = WMI_TLV_CIPHER_NONE, [WMI_CIPHER_WEP] = WMI_TLV_CIPHER_WEP, [WMI_CIPHER_TKIP] = WMI_TLV_CIPHER_TKIP, [WMI_CIPHER_AES_OCB] = WMI_TLV_CIPHER_AES_OCB, [WMI_CIPHER_AES_CCM] = WMI_TLV_CIPHER_AES_CCM, [WMI_CIPHER_WAPI] = WMI_TLV_CIPHER_WAPI, [WMI_CIPHER_CKIP] = WMI_TLV_CIPHER_CKIP, [WMI_CIPHER_AES_CMAC] = WMI_TLV_CIPHER_AES_CMAC, [WMI_CIPHER_AES_GCM] = WMI_TLV_CIPHER_AES_GCM, }; static const struct wmi_peer_flags_map wmi_peer_flags_map = { .auth = WMI_PEER_AUTH, .qos = WMI_PEER_QOS, .need_ptk_4_way = WMI_PEER_NEED_PTK_4_WAY, .need_gtk_2_way = WMI_PEER_NEED_GTK_2_WAY, .apsd = WMI_PEER_APSD, .ht = WMI_PEER_HT, .bw40 = WMI_PEER_40MHZ, .stbc = WMI_PEER_STBC, .ldbc = WMI_PEER_LDPC, .dyn_mimops = WMI_PEER_DYN_MIMOPS, .static_mimops = WMI_PEER_STATIC_MIMOPS, .spatial_mux = WMI_PEER_SPATIAL_MUX, .vht = WMI_PEER_VHT, .bw80 = WMI_PEER_80MHZ, .vht_2g = WMI_PEER_VHT_2G, .pmf = WMI_PEER_PMF, .bw160 = WMI_PEER_160MHZ, }; static const struct wmi_peer_flags_map wmi_10x_peer_flags_map = { .auth = WMI_10X_PEER_AUTH, .qos = WMI_10X_PEER_QOS, .need_ptk_4_way = WMI_10X_PEER_NEED_PTK_4_WAY, .need_gtk_2_way = WMI_10X_PEER_NEED_GTK_2_WAY, .apsd = WMI_10X_PEER_APSD, .ht = WMI_10X_PEER_HT, .bw40 = WMI_10X_PEER_40MHZ, .stbc = WMI_10X_PEER_STBC, .ldbc = WMI_10X_PEER_LDPC, .dyn_mimops = WMI_10X_PEER_DYN_MIMOPS, .static_mimops = WMI_10X_PEER_STATIC_MIMOPS, .spatial_mux = WMI_10X_PEER_SPATIAL_MUX, .vht = WMI_10X_PEER_VHT, .bw80 = WMI_10X_PEER_80MHZ, .bw160 = WMI_10X_PEER_160MHZ, }; static const struct wmi_peer_flags_map wmi_10_2_peer_flags_map = { .auth = WMI_10_2_PEER_AUTH, .qos = WMI_10_2_PEER_QOS, .need_ptk_4_way = WMI_10_2_PEER_NEED_PTK_4_WAY, .need_gtk_2_way = WMI_10_2_PEER_NEED_GTK_2_WAY, .apsd = WMI_10_2_PEER_APSD, .ht = WMI_10_2_PEER_HT, .bw40 = WMI_10_2_PEER_40MHZ, .stbc = WMI_10_2_PEER_STBC, .ldbc = WMI_10_2_PEER_LDPC, .dyn_mimops = WMI_10_2_PEER_DYN_MIMOPS, .static_mimops = WMI_10_2_PEER_STATIC_MIMOPS, .spatial_mux = WMI_10_2_PEER_SPATIAL_MUX, .vht = WMI_10_2_PEER_VHT, .bw80 = WMI_10_2_PEER_80MHZ, .vht_2g = WMI_10_2_PEER_VHT_2G, .pmf = WMI_10_2_PEER_PMF, .bw160 = WMI_10_2_PEER_160MHZ, }; void ath10k_wmi_put_wmi_channel(struct ath10k *ar, struct wmi_channel *ch, const struct wmi_channel_arg *arg) { u32 flags = 0; struct ieee80211_channel *chan = NULL; memset(ch, 0, sizeof(*ch)); if (arg->passive) flags |= WMI_CHAN_FLAG_PASSIVE; if (arg->allow_ibss) flags |= WMI_CHAN_FLAG_ADHOC_ALLOWED; if (arg->allow_ht) flags |= WMI_CHAN_FLAG_ALLOW_HT; if (arg->allow_vht) flags |= WMI_CHAN_FLAG_ALLOW_VHT; if (arg->ht40plus) flags |= WMI_CHAN_FLAG_HT40_PLUS; if (arg->chan_radar) flags |= WMI_CHAN_FLAG_DFS; ch->band_center_freq2 = 0; ch->mhz = __cpu_to_le32(arg->freq); ch->band_center_freq1 = __cpu_to_le32(arg->band_center_freq1); if (arg->mode == MODE_11AC_VHT80_80) { ch->band_center_freq2 = __cpu_to_le32(arg->band_center_freq2); chan = ieee80211_get_channel(ar->hw->wiphy, arg->band_center_freq2 - 10); } if (arg->mode == MODE_11AC_VHT160) { u32 band_center_freq1; u32 band_center_freq2; if (arg->freq > arg->band_center_freq1) { band_center_freq1 = arg->band_center_freq1 + 40; band_center_freq2 = arg->band_center_freq1 - 40; } else { band_center_freq1 = arg->band_center_freq1 - 40; band_center_freq2 = arg->band_center_freq1 + 40; } ch->band_center_freq1 = __cpu_to_le32(band_center_freq1); /* Minus 10 to get a defined 5G channel frequency*/ chan = ieee80211_get_channel(ar->hw->wiphy, band_center_freq2 - 10); /* The center frequency of the entire VHT160 */ ch->band_center_freq2 = __cpu_to_le32(arg->band_center_freq1); } if (chan && chan->flags & IEEE80211_CHAN_RADAR) flags |= WMI_CHAN_FLAG_DFS_CFREQ2; ch->min_power = arg->min_power; ch->max_power = arg->max_power; ch->reg_power = arg->max_reg_power; ch->antenna_max = arg->max_antenna_gain; ch->max_tx_power = arg->max_power; /* mode & flags share storage */ ch->mode = arg->mode; ch->flags |= __cpu_to_le32(flags); } int ath10k_wmi_wait_for_service_ready(struct ath10k *ar) { unsigned long time_left, i; time_left = wait_for_completion_timeout(&ar->wmi.service_ready, WMI_SERVICE_READY_TIMEOUT_HZ); if (!time_left) { /* Sometimes the PCI HIF doesn't receive interrupt * for the service ready message even if the buffer * was completed. PCIe sniffer shows that it's * because the corresponding CE ring doesn't fires * it. Workaround here by polling CE rings once. */ ath10k_warn(ar, "failed to receive service ready completion, polling..\n"); for (i = 0; i < CE_COUNT; i++) ath10k_hif_send_complete_check(ar, i, 1); time_left = wait_for_completion_timeout(&ar->wmi.service_ready, WMI_SERVICE_READY_TIMEOUT_HZ); if (!time_left) { ath10k_warn(ar, "polling timed out\n"); return -ETIMEDOUT; } ath10k_warn(ar, "service ready completion received, continuing normally\n"); } return 0; } int ath10k_wmi_wait_for_unified_ready(struct ath10k *ar) { unsigned long time_left; time_left = wait_for_completion_timeout(&ar->wmi.unified_ready, WMI_UNIFIED_READY_TIMEOUT_HZ); if (!time_left) return -ETIMEDOUT; return 0; } struct sk_buff *ath10k_wmi_alloc_skb(struct ath10k *ar, u32 len) { struct sk_buff *skb; u32 round_len = roundup(len, 4); skb = ath10k_htc_alloc_skb(ar, WMI_SKB_HEADROOM + round_len); if (!skb) return NULL; skb_reserve(skb, WMI_SKB_HEADROOM); if (!IS_ALIGNED((unsigned long)skb->data, 4)) ath10k_warn(ar, "Unaligned WMI skb\n"); skb_put(skb, round_len); memset(skb->data, 0, round_len); return skb; } static void ath10k_wmi_htc_tx_complete(struct ath10k *ar, struct sk_buff *skb) { dev_kfree_skb(skb); } int ath10k_wmi_cmd_send_nowait(struct ath10k *ar, struct sk_buff *skb, u32 cmd_id) { struct ath10k_skb_cb *skb_cb = ATH10K_SKB_CB(skb); struct wmi_cmd_hdr *cmd_hdr; int ret; u32 cmd = 0; if (skb_push(skb, sizeof(struct wmi_cmd_hdr)) == NULL) return -ENOMEM; cmd |= SM(cmd_id, WMI_CMD_HDR_CMD_ID); cmd_hdr = (struct wmi_cmd_hdr *)skb->data; cmd_hdr->cmd_id = __cpu_to_le32(cmd); memset(skb_cb, 0, sizeof(*skb_cb)); trace_ath10k_wmi_cmd(ar, cmd_id, skb->data, skb->len); ret = ath10k_htc_send(&ar->htc, ar->wmi.eid, skb); if (ret) goto err_pull; return 0; err_pull: skb_pull(skb, sizeof(struct wmi_cmd_hdr)); return ret; } static void ath10k_wmi_tx_beacon_nowait(struct ath10k_vif *arvif) { struct ath10k *ar = arvif->ar; struct ath10k_skb_cb *cb; struct sk_buff *bcn; bool dtim_zero; bool deliver_cab; int ret; spin_lock_bh(&ar->data_lock); bcn = arvif->beacon; if (!bcn) goto unlock; cb = ATH10K_SKB_CB(bcn); switch (arvif->beacon_state) { case ATH10K_BEACON_SENDING: case ATH10K_BEACON_SENT: break; case ATH10K_BEACON_SCHEDULED: arvif->beacon_state = ATH10K_BEACON_SENDING; spin_unlock_bh(&ar->data_lock); dtim_zero = !!(cb->flags & ATH10K_SKB_F_DTIM_ZERO); deliver_cab = !!(cb->flags & ATH10K_SKB_F_DELIVER_CAB); ret = ath10k_wmi_beacon_send_ref_nowait(arvif->ar, arvif->vdev_id, bcn->data, bcn->len, cb->paddr, dtim_zero, deliver_cab); spin_lock_bh(&ar->data_lock); if (ret == 0) arvif->beacon_state = ATH10K_BEACON_SENT; else arvif->beacon_state = ATH10K_BEACON_SCHEDULED; } unlock: spin_unlock_bh(&ar->data_lock); } static void ath10k_wmi_tx_beacons_iter(void *data, u8 *mac, struct ieee80211_vif *vif) { struct ath10k_vif *arvif = (void *)vif->drv_priv; ath10k_wmi_tx_beacon_nowait(arvif); } static void ath10k_wmi_tx_beacons_nowait(struct ath10k *ar) { ieee80211_iterate_active_interfaces_atomic(ar->hw, ATH10K_ITER_NORMAL_FLAGS, ath10k_wmi_tx_beacons_iter, NULL); } static void ath10k_wmi_op_ep_tx_credits(struct ath10k *ar) { /* try to send pending beacons first. they take priority */ ath10k_wmi_tx_beacons_nowait(ar); wake_up(&ar->wmi.tx_credits_wq); } int ath10k_wmi_cmd_send(struct ath10k *ar, struct sk_buff *skb, u32 cmd_id) { int ret = -EOPNOTSUPP; might_sleep(); if (cmd_id == WMI_CMD_UNSUPPORTED) { ath10k_warn(ar, "wmi command %d is not supported by firmware\n", cmd_id); return ret; } wait_event_timeout(ar->wmi.tx_credits_wq, ({ /* try to send pending beacons first. they take priority */ ath10k_wmi_tx_beacons_nowait(ar); ret = ath10k_wmi_cmd_send_nowait(ar, skb, cmd_id); if (ret && test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags)) ret = -ESHUTDOWN; (ret != -EAGAIN); }), 3 * HZ); if (ret) dev_kfree_skb_any(skb); if (ret == -EAGAIN) { ath10k_warn(ar, "wmi command %d timeout, restarting hardware\n", cmd_id); ath10k_core_start_recovery(ar); } return ret; } static struct sk_buff * ath10k_wmi_op_gen_mgmt_tx(struct ath10k *ar, struct sk_buff *msdu) { struct ath10k_skb_cb *cb = ATH10K_SKB_CB(msdu); struct ath10k_vif *arvif; struct wmi_mgmt_tx_cmd *cmd; struct ieee80211_hdr *hdr; struct sk_buff *skb; int len; u32 vdev_id; u32 buf_len = msdu->len; u16 fc; const u8 *peer_addr; hdr = (struct ieee80211_hdr *)msdu->data; fc = le16_to_cpu(hdr->frame_control); if (cb->vif) { arvif = (void *)cb->vif->drv_priv; vdev_id = arvif->vdev_id; } else { vdev_id = 0; } if (WARN_ON_ONCE(!ieee80211_is_mgmt(hdr->frame_control))) return ERR_PTR(-EINVAL); len = sizeof(cmd->hdr) + msdu->len; if ((ieee80211_is_action(hdr->frame_control) || ieee80211_is_deauth(hdr->frame_control) || ieee80211_is_disassoc(hdr->frame_control)) && ieee80211_has_protected(hdr->frame_control)) { peer_addr = hdr->addr1; if (is_multicast_ether_addr(peer_addr)) { len += sizeof(struct ieee80211_mmie_16); buf_len += sizeof(struct ieee80211_mmie_16); } else { if (cb->ucast_cipher == WLAN_CIPHER_SUITE_GCMP || cb->ucast_cipher == WLAN_CIPHER_SUITE_GCMP_256) { len += IEEE80211_GCMP_MIC_LEN; buf_len += IEEE80211_GCMP_MIC_LEN; } else { len += IEEE80211_CCMP_MIC_LEN; buf_len += IEEE80211_CCMP_MIC_LEN; } } } len = round_up(len, 4); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_mgmt_tx_cmd *)skb->data; cmd->hdr.vdev_id = __cpu_to_le32(vdev_id); cmd->hdr.tx_rate = 0; cmd->hdr.tx_power = 0; cmd->hdr.buf_len = __cpu_to_le32(buf_len); ether_addr_copy(cmd->hdr.peer_macaddr.addr, ieee80211_get_DA(hdr)); memcpy(cmd->buf, msdu->data, msdu->len); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi mgmt tx skb %pK len %d ftype %02x stype %02x\n", msdu, skb->len, fc & IEEE80211_FCTL_FTYPE, fc & IEEE80211_FCTL_STYPE); trace_ath10k_tx_hdr(ar, skb->data, skb->len); trace_ath10k_tx_payload(ar, skb->data, skb->len); return skb; } static void ath10k_wmi_event_scan_started(struct ath10k *ar) { lockdep_assert_held(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: ath10k_warn(ar, "received scan started event in an invalid scan state: %s (%d)\n", ath10k_scan_state_str(ar->scan.state), ar->scan.state); break; case ATH10K_SCAN_STARTING: ar->scan.state = ATH10K_SCAN_RUNNING; if (ar->scan.is_roc) ieee80211_ready_on_channel(ar->hw); complete(&ar->scan.started); break; } } static void ath10k_wmi_event_scan_start_failed(struct ath10k *ar) { lockdep_assert_held(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: ath10k_warn(ar, "received scan start failed event in an invalid scan state: %s (%d)\n", ath10k_scan_state_str(ar->scan.state), ar->scan.state); break; case ATH10K_SCAN_STARTING: complete(&ar->scan.started); __ath10k_scan_finish(ar); break; } } static void ath10k_wmi_event_scan_completed(struct ath10k *ar) { lockdep_assert_held(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_STARTING: /* One suspected reason scan can be completed while starting is * if firmware fails to deliver all scan events to the host, * e.g. when transport pipe is full. This has been observed * with spectral scan phyerr events starving wmi transport * pipe. In such case the "scan completed" event should be (and * is) ignored by the host as it may be just firmware's scan * state machine recovering. */ ath10k_warn(ar, "received scan completed event in an invalid scan state: %s (%d)\n", ath10k_scan_state_str(ar->scan.state), ar->scan.state); break; case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: __ath10k_scan_finish(ar); break; } } static void ath10k_wmi_event_scan_bss_chan(struct ath10k *ar) { lockdep_assert_held(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_STARTING: ath10k_warn(ar, "received scan bss chan event in an invalid scan state: %s (%d)\n", ath10k_scan_state_str(ar->scan.state), ar->scan.state); break; case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: ar->scan_channel = NULL; break; } } static void ath10k_wmi_event_scan_foreign_chan(struct ath10k *ar, u32 freq) { lockdep_assert_held(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_STARTING: ath10k_warn(ar, "received scan foreign chan event in an invalid scan state: %s (%d)\n", ath10k_scan_state_str(ar->scan.state), ar->scan.state); break; case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: ar->scan_channel = ieee80211_get_channel(ar->hw->wiphy, freq); if (ar->scan.is_roc && ar->scan.roc_freq == freq) complete(&ar->scan.on_channel); break; } } static const char * ath10k_wmi_event_scan_type_str(enum wmi_scan_event_type type, enum wmi_scan_completion_reason reason) { switch (type) { case WMI_SCAN_EVENT_STARTED: return "started"; case WMI_SCAN_EVENT_COMPLETED: switch (reason) { case WMI_SCAN_REASON_COMPLETED: return "completed"; case WMI_SCAN_REASON_CANCELLED: return "completed [cancelled]"; case WMI_SCAN_REASON_PREEMPTED: return "completed [preempted]"; case WMI_SCAN_REASON_TIMEDOUT: return "completed [timedout]"; case WMI_SCAN_REASON_INTERNAL_FAILURE: return "completed [internal err]"; case WMI_SCAN_REASON_MAX: break; } return "completed [unknown]"; case WMI_SCAN_EVENT_BSS_CHANNEL: return "bss channel"; case WMI_SCAN_EVENT_FOREIGN_CHANNEL: return "foreign channel"; case WMI_SCAN_EVENT_DEQUEUED: return "dequeued"; case WMI_SCAN_EVENT_PREEMPTED: return "preempted"; case WMI_SCAN_EVENT_START_FAILED: return "start failed"; case WMI_SCAN_EVENT_RESTARTED: return "restarted"; case WMI_SCAN_EVENT_FOREIGN_CHANNEL_EXIT: return "foreign channel exit"; default: return "unknown"; } } static int ath10k_wmi_op_pull_scan_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_scan_ev_arg *arg) { struct wmi_scan_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->event_type = ev->event_type; arg->reason = ev->reason; arg->channel_freq = ev->channel_freq; arg->scan_req_id = ev->scan_req_id; arg->scan_id = ev->scan_id; arg->vdev_id = ev->vdev_id; return 0; } int ath10k_wmi_event_scan(struct ath10k *ar, struct sk_buff *skb) { struct wmi_scan_ev_arg arg = {}; enum wmi_scan_event_type event_type; enum wmi_scan_completion_reason reason; u32 freq; u32 req_id; u32 scan_id; u32 vdev_id; int ret; ret = ath10k_wmi_pull_scan(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse scan event: %d\n", ret); return ret; } event_type = __le32_to_cpu(arg.event_type); reason = __le32_to_cpu(arg.reason); freq = __le32_to_cpu(arg.channel_freq); req_id = __le32_to_cpu(arg.scan_req_id); scan_id = __le32_to_cpu(arg.scan_id); vdev_id = __le32_to_cpu(arg.vdev_id); spin_lock_bh(&ar->data_lock); ath10k_dbg(ar, ATH10K_DBG_WMI, "scan event %s type %d reason %d freq %d req_id %d scan_id %d vdev_id %d state %s (%d)\n", ath10k_wmi_event_scan_type_str(event_type, reason), event_type, reason, freq, req_id, scan_id, vdev_id, ath10k_scan_state_str(ar->scan.state), ar->scan.state); switch (event_type) { case WMI_SCAN_EVENT_STARTED: ath10k_wmi_event_scan_started(ar); break; case WMI_SCAN_EVENT_COMPLETED: ath10k_wmi_event_scan_completed(ar); break; case WMI_SCAN_EVENT_BSS_CHANNEL: ath10k_wmi_event_scan_bss_chan(ar); break; case WMI_SCAN_EVENT_FOREIGN_CHANNEL: ath10k_wmi_event_scan_foreign_chan(ar, freq); break; case WMI_SCAN_EVENT_START_FAILED: ath10k_warn(ar, "received scan start failure event\n"); ath10k_wmi_event_scan_start_failed(ar); break; case WMI_SCAN_EVENT_DEQUEUED: case WMI_SCAN_EVENT_PREEMPTED: case WMI_SCAN_EVENT_RESTARTED: case WMI_SCAN_EVENT_FOREIGN_CHANNEL_EXIT: default: break; } spin_unlock_bh(&ar->data_lock); return 0; } /* If keys are configured, HW decrypts all frames * with protected bit set. Mark such frames as decrypted. */ static void ath10k_wmi_handle_wep_reauth(struct ath10k *ar, struct sk_buff *skb, struct ieee80211_rx_status *status) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; unsigned int hdrlen; bool peer_key; u8 *addr, keyidx; if (!ieee80211_is_auth(hdr->frame_control) || !ieee80211_has_protected(hdr->frame_control)) return; hdrlen = ieee80211_hdrlen(hdr->frame_control); if (skb->len < (hdrlen + IEEE80211_WEP_IV_LEN)) return; keyidx = skb->data[hdrlen + (IEEE80211_WEP_IV_LEN - 1)] >> WEP_KEYID_SHIFT; addr = ieee80211_get_SA(hdr); spin_lock_bh(&ar->data_lock); peer_key = ath10k_mac_is_peer_wep_key_set(ar, addr, keyidx); spin_unlock_bh(&ar->data_lock); if (peer_key) { ath10k_dbg(ar, ATH10K_DBG_MAC, "mac wep key present for peer %pM\n", addr); status->flag |= RX_FLAG_DECRYPTED; } } static int ath10k_wmi_op_pull_mgmt_rx_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_mgmt_rx_ev_arg *arg) { struct wmi_mgmt_rx_event_v1 *ev_v1; struct wmi_mgmt_rx_event_v2 *ev_v2; struct wmi_mgmt_rx_hdr_v1 *ev_hdr; struct wmi_mgmt_rx_ext_info *ext_info; size_t pull_len; u32 msdu_len; u32 len; if (test_bit(ATH10K_FW_FEATURE_EXT_WMI_MGMT_RX, ar->running_fw->fw_file.fw_features)) { ev_v2 = (struct wmi_mgmt_rx_event_v2 *)skb->data; ev_hdr = &ev_v2->hdr.v1; pull_len = sizeof(*ev_v2); } else { ev_v1 = (struct wmi_mgmt_rx_event_v1 *)skb->data; ev_hdr = &ev_v1->hdr; pull_len = sizeof(*ev_v1); } if (skb->len < pull_len) return -EPROTO; skb_pull(skb, pull_len); arg->channel = ev_hdr->channel; arg->buf_len = ev_hdr->buf_len; arg->status = ev_hdr->status; arg->snr = ev_hdr->snr; arg->phy_mode = ev_hdr->phy_mode; arg->rate = ev_hdr->rate; msdu_len = __le32_to_cpu(arg->buf_len); if (skb->len < msdu_len) return -EPROTO; if (le32_to_cpu(arg->status) & WMI_RX_STATUS_EXT_INFO) { len = ALIGN(le32_to_cpu(arg->buf_len), 4); ext_info = (struct wmi_mgmt_rx_ext_info *)(skb->data + len); memcpy(&arg->ext_info, ext_info, sizeof(struct wmi_mgmt_rx_ext_info)); } /* the WMI buffer might've ended up being padded to 4 bytes due to HTC * trailer with credit update. Trim the excess garbage. */ skb_trim(skb, msdu_len); return 0; } static int ath10k_wmi_10_4_op_pull_mgmt_rx_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_mgmt_rx_ev_arg *arg) { struct wmi_10_4_mgmt_rx_event *ev; struct wmi_10_4_mgmt_rx_hdr *ev_hdr; size_t pull_len; u32 msdu_len; struct wmi_mgmt_rx_ext_info *ext_info; u32 len; ev = (struct wmi_10_4_mgmt_rx_event *)skb->data; ev_hdr = &ev->hdr; pull_len = sizeof(*ev); if (skb->len < pull_len) return -EPROTO; skb_pull(skb, pull_len); arg->channel = ev_hdr->channel; arg->buf_len = ev_hdr->buf_len; arg->status = ev_hdr->status; arg->snr = ev_hdr->snr; arg->phy_mode = ev_hdr->phy_mode; arg->rate = ev_hdr->rate; msdu_len = __le32_to_cpu(arg->buf_len); if (skb->len < msdu_len) return -EPROTO; if (le32_to_cpu(arg->status) & WMI_RX_STATUS_EXT_INFO) { len = ALIGN(le32_to_cpu(arg->buf_len), 4); ext_info = (struct wmi_mgmt_rx_ext_info *)(skb->data + len); memcpy(&arg->ext_info, ext_info, sizeof(struct wmi_mgmt_rx_ext_info)); } /* Make sure bytes added for padding are removed. */ skb_trim(skb, msdu_len); return 0; } static bool ath10k_wmi_rx_is_decrypted(struct ath10k *ar, struct ieee80211_hdr *hdr) { if (!ieee80211_has_protected(hdr->frame_control)) return false; /* FW delivers WEP Shared Auth frame with Protected Bit set and * encrypted payload. However in case of PMF it delivers decrypted * frames with Protected Bit set. */ if (ieee80211_is_auth(hdr->frame_control)) return false; /* qca99x0 based FW delivers broadcast or multicast management frames * (ex: group privacy action frames in mesh) as encrypted payload. */ if (is_multicast_ether_addr(ieee80211_get_DA(hdr)) && ar->hw_params.sw_decrypt_mcast_mgmt) return false; return true; } static int wmi_process_mgmt_tx_comp(struct ath10k *ar, struct mgmt_tx_compl_params *param) { struct ath10k_mgmt_tx_pkt_addr *pkt_addr; struct ath10k_wmi *wmi = &ar->wmi; struct ieee80211_tx_info *info; struct sk_buff *msdu; int ret; spin_lock_bh(&ar->data_lock); pkt_addr = idr_find(&wmi->mgmt_pending_tx, param->desc_id); if (!pkt_addr) { ath10k_warn(ar, "received mgmt tx completion for invalid msdu_id: %d\n", param->desc_id); ret = -ENOENT; goto out; } msdu = pkt_addr->vaddr; dma_unmap_single(ar->dev, pkt_addr->paddr, msdu->len, DMA_TO_DEVICE); info = IEEE80211_SKB_CB(msdu); kfree(pkt_addr); if (param->status) { info->flags &= ~IEEE80211_TX_STAT_ACK; } else { info->flags |= IEEE80211_TX_STAT_ACK; info->status.ack_signal = ATH10K_DEFAULT_NOISE_FLOOR + param->ack_rssi; info->status.flags |= IEEE80211_TX_STATUS_ACK_SIGNAL_VALID; } ieee80211_tx_status_irqsafe(ar->hw, msdu); ret = 0; out: idr_remove(&wmi->mgmt_pending_tx, param->desc_id); spin_unlock_bh(&ar->data_lock); return ret; } int ath10k_wmi_event_mgmt_tx_compl(struct ath10k *ar, struct sk_buff *skb) { struct wmi_tlv_mgmt_tx_compl_ev_arg arg; struct mgmt_tx_compl_params param; int ret; ret = ath10k_wmi_pull_mgmt_tx_compl(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse mgmt comp event: %d\n", ret); return ret; } memset(¶m, 0, sizeof(struct mgmt_tx_compl_params)); param.desc_id = __le32_to_cpu(arg.desc_id); param.status = __le32_to_cpu(arg.status); if (test_bit(WMI_SERVICE_TX_DATA_ACK_RSSI, ar->wmi.svc_map)) param.ack_rssi = __le32_to_cpu(arg.ack_rssi); wmi_process_mgmt_tx_comp(ar, ¶m); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi tlv evnt mgmt tx completion\n"); return 0; } int ath10k_wmi_event_mgmt_tx_bundle_compl(struct ath10k *ar, struct sk_buff *skb) { struct wmi_tlv_mgmt_tx_bundle_compl_ev_arg arg; struct mgmt_tx_compl_params param; u32 num_reports; int i, ret; ret = ath10k_wmi_pull_mgmt_tx_bundle_compl(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse bundle mgmt compl event: %d\n", ret); return ret; } num_reports = __le32_to_cpu(arg.num_reports); for (i = 0; i < num_reports; i++) { memset(¶m, 0, sizeof(struct mgmt_tx_compl_params)); param.desc_id = __le32_to_cpu(arg.desc_ids[i]); param.status = __le32_to_cpu(arg.desc_ids[i]); if (test_bit(WMI_SERVICE_TX_DATA_ACK_RSSI, ar->wmi.svc_map)) param.ack_rssi = __le32_to_cpu(arg.ack_rssi[i]); wmi_process_mgmt_tx_comp(ar, ¶m); } ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi tlv event bundle mgmt tx completion\n"); return 0; } int ath10k_wmi_event_mgmt_rx(struct ath10k *ar, struct sk_buff *skb) { struct wmi_mgmt_rx_ev_arg arg = {}; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); struct ieee80211_hdr *hdr; struct ieee80211_supported_band *sband; u32 rx_status; u32 channel; u32 phy_mode; u32 snr, rssi; u32 rate; u16 fc; int ret, i; ret = ath10k_wmi_pull_mgmt_rx(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse mgmt rx event: %d\n", ret); dev_kfree_skb(skb); return ret; } channel = __le32_to_cpu(arg.channel); rx_status = __le32_to_cpu(arg.status); snr = __le32_to_cpu(arg.snr); phy_mode = __le32_to_cpu(arg.phy_mode); rate = __le32_to_cpu(arg.rate); memset(status, 0, sizeof(*status)); ath10k_dbg(ar, ATH10K_DBG_MGMT, "event mgmt rx status %08x\n", rx_status); if ((test_bit(ATH10K_CAC_RUNNING, &ar->dev_flags)) || (rx_status & (WMI_RX_STATUS_ERR_DECRYPT | WMI_RX_STATUS_ERR_KEY_CACHE_MISS | WMI_RX_STATUS_ERR_CRC))) { dev_kfree_skb(skb); return 0; } if (rx_status & WMI_RX_STATUS_ERR_MIC) status->flag |= RX_FLAG_MMIC_ERROR; if (rx_status & WMI_RX_STATUS_EXT_INFO) { status->mactime = __le64_to_cpu(arg.ext_info.rx_mac_timestamp); status->flag |= RX_FLAG_MACTIME_END; } /* Hardware can Rx CCK rates on 5GHz. In that case phy_mode is set to * MODE_11B. This means phy_mode is not a reliable source for the band * of mgmt rx. */ if (channel >= 1 && channel <= 14) { status->band = NL80211_BAND_2GHZ; } else if (channel >= 36 && channel <= ATH10K_MAX_5G_CHAN) { status->band = NL80211_BAND_5GHZ; } else { /* Shouldn't happen unless list of advertised channels to * mac80211 has been changed. */ WARN_ON_ONCE(1); dev_kfree_skb(skb); return 0; } if (phy_mode == MODE_11B && status->band == NL80211_BAND_5GHZ) ath10k_dbg(ar, ATH10K_DBG_MGMT, "wmi mgmt rx 11b (CCK) on 5GHz\n"); sband = &ar->mac.sbands[status->band]; status->freq = ieee80211_channel_to_frequency(channel, status->band); status->signal = snr + ATH10K_DEFAULT_NOISE_FLOOR; BUILD_BUG_ON(ARRAY_SIZE(status->chain_signal) != ARRAY_SIZE(arg.rssi)); for (i = 0; i < ARRAY_SIZE(status->chain_signal); i++) { status->chains &= ~BIT(i); rssi = __le32_to_cpu(arg.rssi[i]); ath10k_dbg(ar, ATH10K_DBG_MGMT, "mgmt rssi[%d]:%d\n", i, arg.rssi[i]); if (rssi != ATH10K_INVALID_RSSI && rssi != 0) { status->chain_signal[i] = ATH10K_DEFAULT_NOISE_FLOOR + rssi; status->chains |= BIT(i); } } status->rate_idx = ath10k_mac_bitrate_to_idx(sband, rate / 100); hdr = (struct ieee80211_hdr *)skb->data; fc = le16_to_cpu(hdr->frame_control); /* Firmware is guaranteed to report all essential management frames via * WMI while it can deliver some extra via HTT. Since there can be * duplicates split the reporting wrt monitor/sniffing. */ status->flag |= RX_FLAG_SKIP_MONITOR; ath10k_wmi_handle_wep_reauth(ar, skb, status); if (ath10k_wmi_rx_is_decrypted(ar, hdr)) { status->flag |= RX_FLAG_DECRYPTED; if (!ieee80211_is_action(hdr->frame_control) && !ieee80211_is_deauth(hdr->frame_control) && !ieee80211_is_disassoc(hdr->frame_control)) { status->flag |= RX_FLAG_IV_STRIPPED | RX_FLAG_MMIC_STRIPPED; hdr->frame_control = __cpu_to_le16(fc & ~IEEE80211_FCTL_PROTECTED); } } if (ieee80211_is_beacon(hdr->frame_control)) ath10k_mac_handle_beacon(ar, skb); if (ieee80211_is_beacon(hdr->frame_control) || ieee80211_is_probe_resp(hdr->frame_control)) status->boottime_ns = ktime_get_boottime_ns(); ath10k_dbg(ar, ATH10K_DBG_MGMT, "event mgmt rx skb %pK len %d ftype %02x stype %02x\n", skb, skb->len, fc & IEEE80211_FCTL_FTYPE, fc & IEEE80211_FCTL_STYPE); ath10k_dbg(ar, ATH10K_DBG_MGMT, "event mgmt rx freq %d band %d snr %d, rate_idx %d\n", status->freq, status->band, status->signal, status->rate_idx); ieee80211_rx_ni(ar->hw, skb); return 0; } static int freq_to_idx(struct ath10k *ar, int freq) { struct ieee80211_supported_band *sband; int band, ch, idx = 0; for (band = NL80211_BAND_2GHZ; band < NUM_NL80211_BANDS; band++) { sband = ar->hw->wiphy->bands[band]; if (!sband) continue; for (ch = 0; ch < sband->n_channels; ch++, idx++) if (sband->channels[ch].center_freq == freq) goto exit; } exit: return idx; } static int ath10k_wmi_op_pull_ch_info_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_ch_info_ev_arg *arg) { struct wmi_chan_info_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->err_code = ev->err_code; arg->freq = ev->freq; arg->cmd_flags = ev->cmd_flags; arg->noise_floor = ev->noise_floor; arg->rx_clear_count = ev->rx_clear_count; arg->cycle_count = ev->cycle_count; return 0; } static int ath10k_wmi_10_4_op_pull_ch_info_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_ch_info_ev_arg *arg) { struct wmi_10_4_chan_info_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->err_code = ev->err_code; arg->freq = ev->freq; arg->cmd_flags = ev->cmd_flags; arg->noise_floor = ev->noise_floor; arg->rx_clear_count = ev->rx_clear_count; arg->cycle_count = ev->cycle_count; arg->chan_tx_pwr_range = ev->chan_tx_pwr_range; arg->chan_tx_pwr_tp = ev->chan_tx_pwr_tp; arg->rx_frame_count = ev->rx_frame_count; return 0; } /* * Handle the channel info event for firmware which only sends one * chan_info event per scanned channel. */ static void ath10k_wmi_event_chan_info_unpaired(struct ath10k *ar, struct chan_info_params *params) { struct survey_info *survey; int idx; if (params->cmd_flags & WMI_CHAN_INFO_FLAG_COMPLETE) { ath10k_dbg(ar, ATH10K_DBG_WMI, "chan info report completed\n"); return; } idx = freq_to_idx(ar, params->freq); if (idx >= ARRAY_SIZE(ar->survey)) { ath10k_warn(ar, "chan info: invalid frequency %d (idx %d out of bounds)\n", params->freq, idx); return; } survey = &ar->survey[idx]; if (!params->mac_clk_mhz) return; memset(survey, 0, sizeof(*survey)); survey->noise = params->noise_floor; survey->time = (params->cycle_count / params->mac_clk_mhz) / 1000; survey->time_busy = (params->rx_clear_count / params->mac_clk_mhz) / 1000; survey->filled |= SURVEY_INFO_NOISE_DBM | SURVEY_INFO_TIME | SURVEY_INFO_TIME_BUSY; } /* * Handle the channel info event for firmware which sends chan_info * event in pairs(start and stop events) for every scanned channel. */ static void ath10k_wmi_event_chan_info_paired(struct ath10k *ar, struct chan_info_params *params) { struct survey_info *survey; int idx; idx = freq_to_idx(ar, params->freq); if (idx >= ARRAY_SIZE(ar->survey)) { ath10k_warn(ar, "chan info: invalid frequency %d (idx %d out of bounds)\n", params->freq, idx); return; } if (params->cmd_flags & WMI_CHAN_INFO_FLAG_COMPLETE) { if (ar->ch_info_can_report_survey) { survey = &ar->survey[idx]; survey->noise = params->noise_floor; survey->filled = SURVEY_INFO_NOISE_DBM; ath10k_hw_fill_survey_time(ar, survey, params->cycle_count, params->rx_clear_count, ar->survey_last_cycle_count, ar->survey_last_rx_clear_count); } ar->ch_info_can_report_survey = false; } else { ar->ch_info_can_report_survey = true; } if (!(params->cmd_flags & WMI_CHAN_INFO_FLAG_PRE_COMPLETE)) { ar->survey_last_rx_clear_count = params->rx_clear_count; ar->survey_last_cycle_count = params->cycle_count; } } void ath10k_wmi_event_chan_info(struct ath10k *ar, struct sk_buff *skb) { struct chan_info_params ch_info_param; struct wmi_ch_info_ev_arg arg = {}; int ret; ret = ath10k_wmi_pull_ch_info(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse chan info event: %d\n", ret); return; } ch_info_param.err_code = __le32_to_cpu(arg.err_code); ch_info_param.freq = __le32_to_cpu(arg.freq); ch_info_param.cmd_flags = __le32_to_cpu(arg.cmd_flags); ch_info_param.noise_floor = __le32_to_cpu(arg.noise_floor); ch_info_param.rx_clear_count = __le32_to_cpu(arg.rx_clear_count); ch_info_param.cycle_count = __le32_to_cpu(arg.cycle_count); ch_info_param.mac_clk_mhz = __le32_to_cpu(arg.mac_clk_mhz); ath10k_dbg(ar, ATH10K_DBG_WMI, "chan info err_code %d freq %d cmd_flags %d noise_floor %d rx_clear_count %d cycle_count %d\n", ch_info_param.err_code, ch_info_param.freq, ch_info_param.cmd_flags, ch_info_param.noise_floor, ch_info_param.rx_clear_count, ch_info_param.cycle_count); spin_lock_bh(&ar->data_lock); switch (ar->scan.state) { case ATH10K_SCAN_IDLE: case ATH10K_SCAN_STARTING: ath10k_dbg(ar, ATH10K_DBG_WMI, "received chan info event without a scan request, ignoring\n"); goto exit; case ATH10K_SCAN_RUNNING: case ATH10K_SCAN_ABORTING: break; } if (test_bit(ATH10K_FW_FEATURE_SINGLE_CHAN_INFO_PER_CHANNEL, ar->running_fw->fw_file.fw_features)) ath10k_wmi_event_chan_info_unpaired(ar, &ch_info_param); else ath10k_wmi_event_chan_info_paired(ar, &ch_info_param); exit: spin_unlock_bh(&ar->data_lock); } void ath10k_wmi_event_echo(struct ath10k *ar, struct sk_buff *skb) { struct wmi_echo_ev_arg arg = {}; int ret; ret = ath10k_wmi_pull_echo_ev(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse echo: %d\n", ret); return; } ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event echo value 0x%08x\n", le32_to_cpu(arg.value)); if (le32_to_cpu(arg.value) == ATH10K_WMI_BARRIER_ECHO_ID) complete(&ar->wmi.barrier); } int ath10k_wmi_event_debug_mesg(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event debug mesg len %d\n", skb->len); trace_ath10k_wmi_dbglog(ar, skb->data, skb->len); return 0; } void ath10k_wmi_pull_pdev_stats_base(const struct wmi_pdev_stats_base *src, struct ath10k_fw_stats_pdev *dst) { dst->ch_noise_floor = __le32_to_cpu(src->chan_nf); dst->tx_frame_count = __le32_to_cpu(src->tx_frame_count); dst->rx_frame_count = __le32_to_cpu(src->rx_frame_count); dst->rx_clear_count = __le32_to_cpu(src->rx_clear_count); dst->cycle_count = __le32_to_cpu(src->cycle_count); dst->phy_err_count = __le32_to_cpu(src->phy_err_count); dst->chan_tx_power = __le32_to_cpu(src->chan_tx_pwr); } void ath10k_wmi_pull_pdev_stats_tx(const struct wmi_pdev_stats_tx *src, struct ath10k_fw_stats_pdev *dst) { dst->comp_queued = __le32_to_cpu(src->comp_queued); dst->comp_delivered = __le32_to_cpu(src->comp_delivered); dst->msdu_enqued = __le32_to_cpu(src->msdu_enqued); dst->mpdu_enqued = __le32_to_cpu(src->mpdu_enqued); dst->wmm_drop = __le32_to_cpu(src->wmm_drop); dst->local_enqued = __le32_to_cpu(src->local_enqued); dst->local_freed = __le32_to_cpu(src->local_freed); dst->hw_queued = __le32_to_cpu(src->hw_queued); dst->hw_reaped = __le32_to_cpu(src->hw_reaped); dst->underrun = __le32_to_cpu(src->underrun); dst->tx_abort = __le32_to_cpu(src->tx_abort); dst->mpdus_requeued = __le32_to_cpu(src->mpdus_requeued); dst->tx_ko = __le32_to_cpu(src->tx_ko); dst->data_rc = __le32_to_cpu(src->data_rc); dst->self_triggers = __le32_to_cpu(src->self_triggers); dst->sw_retry_failure = __le32_to_cpu(src->sw_retry_failure); dst->illgl_rate_phy_err = __le32_to_cpu(src->illgl_rate_phy_err); dst->pdev_cont_xretry = __le32_to_cpu(src->pdev_cont_xretry); dst->pdev_tx_timeout = __le32_to_cpu(src->pdev_tx_timeout); dst->pdev_resets = __le32_to_cpu(src->pdev_resets); dst->phy_underrun = __le32_to_cpu(src->phy_underrun); dst->txop_ovf = __le32_to_cpu(src->txop_ovf); } static void ath10k_wmi_10_4_pull_pdev_stats_tx(const struct wmi_10_4_pdev_stats_tx *src, struct ath10k_fw_stats_pdev *dst) { dst->comp_queued = __le32_to_cpu(src->comp_queued); dst->comp_delivered = __le32_to_cpu(src->comp_delivered); dst->msdu_enqued = __le32_to_cpu(src->msdu_enqued); dst->mpdu_enqued = __le32_to_cpu(src->mpdu_enqued); dst->wmm_drop = __le32_to_cpu(src->wmm_drop); dst->local_enqued = __le32_to_cpu(src->local_enqued); dst->local_freed = __le32_to_cpu(src->local_freed); dst->hw_queued = __le32_to_cpu(src->hw_queued); dst->hw_reaped = __le32_to_cpu(src->hw_reaped); dst->underrun = __le32_to_cpu(src->underrun); dst->tx_abort = __le32_to_cpu(src->tx_abort); dst->mpdus_requeued = __le32_to_cpu(src->mpdus_requeued); dst->tx_ko = __le32_to_cpu(src->tx_ko); dst->data_rc = __le32_to_cpu(src->data_rc); dst->self_triggers = __le32_to_cpu(src->self_triggers); dst->sw_retry_failure = __le32_to_cpu(src->sw_retry_failure); dst->illgl_rate_phy_err = __le32_to_cpu(src->illgl_rate_phy_err); dst->pdev_cont_xretry = __le32_to_cpu(src->pdev_cont_xretry); dst->pdev_tx_timeout = __le32_to_cpu(src->pdev_tx_timeout); dst->pdev_resets = __le32_to_cpu(src->pdev_resets); dst->phy_underrun = __le32_to_cpu(src->phy_underrun); dst->txop_ovf = __le32_to_cpu(src->txop_ovf); dst->hw_paused = __le32_to_cpu(src->hw_paused); dst->seq_posted = __le32_to_cpu(src->seq_posted); dst->seq_failed_queueing = __le32_to_cpu(src->seq_failed_queueing); dst->seq_completed = __le32_to_cpu(src->seq_completed); dst->seq_restarted = __le32_to_cpu(src->seq_restarted); dst->mu_seq_posted = __le32_to_cpu(src->mu_seq_posted); dst->mpdus_sw_flush = __le32_to_cpu(src->mpdus_sw_flush); dst->mpdus_hw_filter = __le32_to_cpu(src->mpdus_hw_filter); dst->mpdus_truncated = __le32_to_cpu(src->mpdus_truncated); dst->mpdus_ack_failed = __le32_to_cpu(src->mpdus_ack_failed); dst->mpdus_hw_filter = __le32_to_cpu(src->mpdus_hw_filter); dst->mpdus_expired = __le32_to_cpu(src->mpdus_expired); } void ath10k_wmi_pull_pdev_stats_rx(const struct wmi_pdev_stats_rx *src, struct ath10k_fw_stats_pdev *dst) { dst->mid_ppdu_route_change = __le32_to_cpu(src->mid_ppdu_route_change); dst->status_rcvd = __le32_to_cpu(src->status_rcvd); dst->r0_frags = __le32_to_cpu(src->r0_frags); dst->r1_frags = __le32_to_cpu(src->r1_frags); dst->r2_frags = __le32_to_cpu(src->r2_frags); dst->r3_frags = __le32_to_cpu(src->r3_frags); dst->htt_msdus = __le32_to_cpu(src->htt_msdus); dst->htt_mpdus = __le32_to_cpu(src->htt_mpdus); dst->loc_msdus = __le32_to_cpu(src->loc_msdus); dst->loc_mpdus = __le32_to_cpu(src->loc_mpdus); dst->oversize_amsdu = __le32_to_cpu(src->oversize_amsdu); dst->phy_errs = __le32_to_cpu(src->phy_errs); dst->phy_err_drop = __le32_to_cpu(src->phy_err_drop); dst->mpdu_errs = __le32_to_cpu(src->mpdu_errs); } void ath10k_wmi_pull_pdev_stats_extra(const struct wmi_pdev_stats_extra *src, struct ath10k_fw_stats_pdev *dst) { dst->ack_rx_bad = __le32_to_cpu(src->ack_rx_bad); dst->rts_bad = __le32_to_cpu(src->rts_bad); dst->rts_good = __le32_to_cpu(src->rts_good); dst->fcs_bad = __le32_to_cpu(src->fcs_bad); dst->no_beacons = __le32_to_cpu(src->no_beacons); dst->mib_int_count = __le32_to_cpu(src->mib_int_count); } void ath10k_wmi_pull_peer_stats(const struct wmi_peer_stats *src, struct ath10k_fw_stats_peer *dst) { ether_addr_copy(dst->peer_macaddr, src->peer_macaddr.addr); dst->peer_rssi = __le32_to_cpu(src->peer_rssi); dst->peer_tx_rate = __le32_to_cpu(src->peer_tx_rate); } static void ath10k_wmi_10_4_pull_peer_stats(const struct wmi_10_4_peer_stats *src, struct ath10k_fw_stats_peer *dst) { ether_addr_copy(dst->peer_macaddr, src->peer_macaddr.addr); dst->peer_rssi = __le32_to_cpu(src->peer_rssi); dst->peer_tx_rate = __le32_to_cpu(src->peer_tx_rate); dst->peer_rx_rate = __le32_to_cpu(src->peer_rx_rate); } static void ath10k_wmi_10_4_pull_vdev_stats(const struct wmi_vdev_stats_extd *src, struct ath10k_fw_stats_vdev_extd *dst) { dst->vdev_id = __le32_to_cpu(src->vdev_id); dst->ppdu_aggr_cnt = __le32_to_cpu(src->ppdu_aggr_cnt); dst->ppdu_noack = __le32_to_cpu(src->ppdu_noack); dst->mpdu_queued = __le32_to_cpu(src->mpdu_queued); dst->ppdu_nonaggr_cnt = __le32_to_cpu(src->ppdu_nonaggr_cnt); dst->mpdu_sw_requeued = __le32_to_cpu(src->mpdu_sw_requeued); dst->mpdu_suc_retry = __le32_to_cpu(src->mpdu_suc_retry); dst->mpdu_suc_multitry = __le32_to_cpu(src->mpdu_suc_multitry); dst->mpdu_fail_retry = __le32_to_cpu(src->mpdu_fail_retry); dst->tx_ftm_suc = __le32_to_cpu(src->tx_ftm_suc); dst->tx_ftm_suc_retry = __le32_to_cpu(src->tx_ftm_suc_retry); dst->tx_ftm_fail = __le32_to_cpu(src->tx_ftm_fail); dst->rx_ftmr_cnt = __le32_to_cpu(src->rx_ftmr_cnt); dst->rx_ftmr_dup_cnt = __le32_to_cpu(src->rx_ftmr_dup_cnt); dst->rx_iftmr_cnt = __le32_to_cpu(src->rx_iftmr_cnt); dst->rx_iftmr_dup_cnt = __le32_to_cpu(src->rx_iftmr_dup_cnt); } static int ath10k_wmi_main_op_pull_fw_stats(struct ath10k *ar, struct sk_buff *skb, struct ath10k_fw_stats *stats) { const struct wmi_stats_event *ev = (void *)skb->data; u32 num_pdev_stats, num_peer_stats; int i; if (!skb_pull(skb, sizeof(*ev))) return -EPROTO; num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats); num_peer_stats = __le32_to_cpu(ev->num_peer_stats); for (i = 0; i < num_pdev_stats; i++) { const struct wmi_pdev_stats *src; struct ath10k_fw_stats_pdev *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_pdev_stats_base(&src->base, dst); ath10k_wmi_pull_pdev_stats_tx(&src->tx, dst); ath10k_wmi_pull_pdev_stats_rx(&src->rx, dst); list_add_tail(&dst->list, &stats->pdevs); } /* fw doesn't implement vdev stats */ for (i = 0; i < num_peer_stats; i++) { const struct wmi_peer_stats *src; struct ath10k_fw_stats_peer *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_peer_stats(src, dst); list_add_tail(&dst->list, &stats->peers); } return 0; } static int ath10k_wmi_10x_op_pull_fw_stats(struct ath10k *ar, struct sk_buff *skb, struct ath10k_fw_stats *stats) { const struct wmi_stats_event *ev = (void *)skb->data; u32 num_pdev_stats, num_peer_stats; int i; if (!skb_pull(skb, sizeof(*ev))) return -EPROTO; num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats); num_peer_stats = __le32_to_cpu(ev->num_peer_stats); for (i = 0; i < num_pdev_stats; i++) { const struct wmi_10x_pdev_stats *src; struct ath10k_fw_stats_pdev *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_pdev_stats_base(&src->base, dst); ath10k_wmi_pull_pdev_stats_tx(&src->tx, dst); ath10k_wmi_pull_pdev_stats_rx(&src->rx, dst); ath10k_wmi_pull_pdev_stats_extra(&src->extra, dst); list_add_tail(&dst->list, &stats->pdevs); } /* fw doesn't implement vdev stats */ for (i = 0; i < num_peer_stats; i++) { const struct wmi_10x_peer_stats *src; struct ath10k_fw_stats_peer *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_peer_stats(&src->old, dst); dst->peer_rx_rate = __le32_to_cpu(src->peer_rx_rate); list_add_tail(&dst->list, &stats->peers); } return 0; } static int ath10k_wmi_10_2_op_pull_fw_stats(struct ath10k *ar, struct sk_buff *skb, struct ath10k_fw_stats *stats) { const struct wmi_10_2_stats_event *ev = (void *)skb->data; u32 num_pdev_stats; u32 num_pdev_ext_stats; u32 num_peer_stats; int i; if (!skb_pull(skb, sizeof(*ev))) return -EPROTO; num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats); num_pdev_ext_stats = __le32_to_cpu(ev->num_pdev_ext_stats); num_peer_stats = __le32_to_cpu(ev->num_peer_stats); for (i = 0; i < num_pdev_stats; i++) { const struct wmi_10_2_pdev_stats *src; struct ath10k_fw_stats_pdev *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_pdev_stats_base(&src->base, dst); ath10k_wmi_pull_pdev_stats_tx(&src->tx, dst); ath10k_wmi_pull_pdev_stats_rx(&src->rx, dst); ath10k_wmi_pull_pdev_stats_extra(&src->extra, dst); /* FIXME: expose 10.2 specific values */ list_add_tail(&dst->list, &stats->pdevs); } for (i = 0; i < num_pdev_ext_stats; i++) { const struct wmi_10_2_pdev_ext_stats *src; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; /* FIXME: expose values to userspace * * Note: Even though this loop seems to do nothing it is * required to parse following sub-structures properly. */ } /* fw doesn't implement vdev stats */ for (i = 0; i < num_peer_stats; i++) { const struct wmi_10_2_peer_stats *src; struct ath10k_fw_stats_peer *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_peer_stats(&src->old, dst); dst->peer_rx_rate = __le32_to_cpu(src->peer_rx_rate); /* FIXME: expose 10.2 specific values */ list_add_tail(&dst->list, &stats->peers); } return 0; } static int ath10k_wmi_10_2_4_op_pull_fw_stats(struct ath10k *ar, struct sk_buff *skb, struct ath10k_fw_stats *stats) { const struct wmi_10_2_stats_event *ev = (void *)skb->data; u32 num_pdev_stats; u32 num_pdev_ext_stats; u32 num_peer_stats; int i; if (!skb_pull(skb, sizeof(*ev))) return -EPROTO; num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats); num_pdev_ext_stats = __le32_to_cpu(ev->num_pdev_ext_stats); num_peer_stats = __le32_to_cpu(ev->num_peer_stats); for (i = 0; i < num_pdev_stats; i++) { const struct wmi_10_2_pdev_stats *src; struct ath10k_fw_stats_pdev *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_pdev_stats_base(&src->base, dst); ath10k_wmi_pull_pdev_stats_tx(&src->tx, dst); ath10k_wmi_pull_pdev_stats_rx(&src->rx, dst); ath10k_wmi_pull_pdev_stats_extra(&src->extra, dst); /* FIXME: expose 10.2 specific values */ list_add_tail(&dst->list, &stats->pdevs); } for (i = 0; i < num_pdev_ext_stats; i++) { const struct wmi_10_2_pdev_ext_stats *src; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; /* FIXME: expose values to userspace * * Note: Even though this loop seems to do nothing it is * required to parse following sub-structures properly. */ } /* fw doesn't implement vdev stats */ for (i = 0; i < num_peer_stats; i++) { const struct wmi_10_2_4_ext_peer_stats *src; struct ath10k_fw_stats_peer *dst; int stats_len; if (test_bit(WMI_SERVICE_PEER_STATS, ar->wmi.svc_map)) stats_len = sizeof(struct wmi_10_2_4_ext_peer_stats); else stats_len = sizeof(struct wmi_10_2_4_peer_stats); src = (void *)skb->data; if (!skb_pull(skb, stats_len)) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_peer_stats(&src->common.old, dst); dst->peer_rx_rate = __le32_to_cpu(src->common.peer_rx_rate); if (ath10k_peer_stats_enabled(ar)) dst->rx_duration = __le32_to_cpu(src->rx_duration); /* FIXME: expose 10.2 specific values */ list_add_tail(&dst->list, &stats->peers); } return 0; } static int ath10k_wmi_10_4_op_pull_fw_stats(struct ath10k *ar, struct sk_buff *skb, struct ath10k_fw_stats *stats) { const struct wmi_10_2_stats_event *ev = (void *)skb->data; u32 num_pdev_stats; u32 num_pdev_ext_stats; u32 num_vdev_stats; u32 num_peer_stats; u32 num_bcnflt_stats; u32 stats_id; int i; if (!skb_pull(skb, sizeof(*ev))) return -EPROTO; num_pdev_stats = __le32_to_cpu(ev->num_pdev_stats); num_pdev_ext_stats = __le32_to_cpu(ev->num_pdev_ext_stats); num_vdev_stats = __le32_to_cpu(ev->num_vdev_stats); num_peer_stats = __le32_to_cpu(ev->num_peer_stats); num_bcnflt_stats = __le32_to_cpu(ev->num_bcnflt_stats); stats_id = __le32_to_cpu(ev->stats_id); for (i = 0; i < num_pdev_stats; i++) { const struct wmi_10_4_pdev_stats *src; struct ath10k_fw_stats_pdev *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_pull_pdev_stats_base(&src->base, dst); ath10k_wmi_10_4_pull_pdev_stats_tx(&src->tx, dst); ath10k_wmi_pull_pdev_stats_rx(&src->rx, dst); dst->rx_ovfl_errs = __le32_to_cpu(src->rx_ovfl_errs); ath10k_wmi_pull_pdev_stats_extra(&src->extra, dst); list_add_tail(&dst->list, &stats->pdevs); } for (i = 0; i < num_pdev_ext_stats; i++) { const struct wmi_10_2_pdev_ext_stats *src; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; /* FIXME: expose values to userspace * * Note: Even though this loop seems to do nothing it is * required to parse following sub-structures properly. */ } for (i = 0; i < num_vdev_stats; i++) { const struct wmi_vdev_stats *src; /* Ignore vdev stats here as it has only vdev id. Actual vdev * stats will be retrieved from vdev extended stats. */ src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; } for (i = 0; i < num_peer_stats; i++) { const struct wmi_10_4_peer_stats *src; struct ath10k_fw_stats_peer *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_10_4_pull_peer_stats(src, dst); list_add_tail(&dst->list, &stats->peers); } for (i = 0; i < num_bcnflt_stats; i++) { const struct wmi_10_4_bss_bcn_filter_stats *src; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; /* FIXME: expose values to userspace * * Note: Even though this loop seems to do nothing it is * required to parse following sub-structures properly. */ } if (stats_id & WMI_10_4_STAT_PEER_EXTD) { stats->extended = true; for (i = 0; i < num_peer_stats; i++) { const struct wmi_10_4_peer_extd_stats *src; struct ath10k_fw_extd_stats_peer *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ether_addr_copy(dst->peer_macaddr, src->peer_macaddr.addr); dst->rx_duration = __le32_to_cpu(src->rx_duration); list_add_tail(&dst->list, &stats->peers_extd); } } if (stats_id & WMI_10_4_STAT_VDEV_EXTD) { for (i = 0; i < num_vdev_stats; i++) { const struct wmi_vdev_stats_extd *src; struct ath10k_fw_stats_vdev_extd *dst; src = (void *)skb->data; if (!skb_pull(skb, sizeof(*src))) return -EPROTO; dst = kzalloc(sizeof(*dst), GFP_ATOMIC); if (!dst) continue; ath10k_wmi_10_4_pull_vdev_stats(src, dst); list_add_tail(&dst->list, &stats->vdevs); } } return 0; } void ath10k_wmi_event_update_stats(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_UPDATE_STATS_EVENTID\n"); ath10k_debug_fw_stats_process(ar, skb); } static int ath10k_wmi_op_pull_vdev_start_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_vdev_start_ev_arg *arg) { struct wmi_vdev_start_response_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->vdev_id = ev->vdev_id; arg->req_id = ev->req_id; arg->resp_type = ev->resp_type; arg->status = ev->status; return 0; } void ath10k_wmi_event_vdev_start_resp(struct ath10k *ar, struct sk_buff *skb) { struct wmi_vdev_start_ev_arg arg = {}; int ret; u32 status; ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_VDEV_START_RESP_EVENTID\n"); ar->last_wmi_vdev_start_status = 0; ret = ath10k_wmi_pull_vdev_start(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse vdev start event: %d\n", ret); ar->last_wmi_vdev_start_status = ret; goto out; } status = __le32_to_cpu(arg.status); if (WARN_ON_ONCE(status)) { ath10k_warn(ar, "vdev-start-response reports status error: %d (%s)\n", status, (status == WMI_VDEV_START_CHAN_INVALID) ? "chan-invalid" : "unknown"); /* Setup is done one way or another though, so we should still * do the completion, so don't return here. */ ar->last_wmi_vdev_start_status = -EINVAL; } out: complete(&ar->vdev_setup_done); } void ath10k_wmi_event_vdev_stopped(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_VDEV_STOPPED_EVENTID\n"); complete(&ar->vdev_setup_done); } static int ath10k_wmi_op_pull_peer_kick_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_peer_kick_ev_arg *arg) { struct wmi_peer_sta_kickout_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->mac_addr = ev->peer_macaddr.addr; return 0; } void ath10k_wmi_event_peer_sta_kickout(struct ath10k *ar, struct sk_buff *skb) { struct wmi_peer_kick_ev_arg arg = {}; struct ieee80211_sta *sta; int ret; ret = ath10k_wmi_pull_peer_kick(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse peer kickout event: %d\n", ret); return; } ath10k_dbg(ar, ATH10K_DBG_STA, "wmi event peer sta kickout %pM\n", arg.mac_addr); rcu_read_lock(); sta = ieee80211_find_sta_by_ifaddr(ar->hw, arg.mac_addr, NULL); if (!sta) { ath10k_warn(ar, "Spurious quick kickout for STA %pM\n", arg.mac_addr); goto exit; } ieee80211_report_low_ack(sta, 10); exit: rcu_read_unlock(); } /* * FIXME * * We don't report to mac80211 sleep state of connected * stations. Due to this mac80211 can't fill in TIM IE * correctly. * * I know of no way of getting nullfunc frames that contain * sleep transition from connected stations - these do not * seem to be sent from the target to the host. There also * doesn't seem to be a dedicated event for that. So the * only way left to do this would be to read tim_bitmap * during SWBA. * * We could probably try using tim_bitmap from SWBA to tell * mac80211 which stations are asleep and which are not. The * problem here is calling mac80211 functions so many times * could take too long and make us miss the time to submit * the beacon to the target. * * So as a workaround we try to extend the TIM IE if there * is unicast buffered for stations with aid > 7 and fill it * in ourselves. */ static void ath10k_wmi_update_tim(struct ath10k *ar, struct ath10k_vif *arvif, struct sk_buff *bcn, const struct wmi_tim_info_arg *tim_info) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)bcn->data; struct ieee80211_tim_ie *tim; u8 *ies, *ie; u8 ie_len, pvm_len; __le32 t; u32 v, tim_len; /* When FW reports 0 in tim_len, ensure at least first byte * in tim_bitmap is considered for pvm calculation. */ tim_len = tim_info->tim_len ? __le32_to_cpu(tim_info->tim_len) : 1; /* if next SWBA has no tim_changed the tim_bitmap is garbage. * we must copy the bitmap upon change and reuse it later */ if (__le32_to_cpu(tim_info->tim_changed)) { int i; if (sizeof(arvif->u.ap.tim_bitmap) < tim_len) { ath10k_warn(ar, "SWBA TIM field is too big (%u), truncated it to %zu", tim_len, sizeof(arvif->u.ap.tim_bitmap)); tim_len = sizeof(arvif->u.ap.tim_bitmap); } for (i = 0; i < tim_len; i++) { t = tim_info->tim_bitmap[i / 4]; v = __le32_to_cpu(t); arvif->u.ap.tim_bitmap[i] = (v >> ((i % 4) * 8)) & 0xFF; } /* FW reports either length 0 or length based on max supported * station. so we calculate this on our own */ arvif->u.ap.tim_len = 0; for (i = 0; i < tim_len; i++) if (arvif->u.ap.tim_bitmap[i]) arvif->u.ap.tim_len = i; arvif->u.ap.tim_len++; } ies = bcn->data; ies += ieee80211_hdrlen(hdr->frame_control); ies += 12; /* fixed parameters */ ie = (u8 *)cfg80211_find_ie(WLAN_EID_TIM, ies, (u8 *)skb_tail_pointer(bcn) - ies); if (!ie) { if (arvif->vdev_type != WMI_VDEV_TYPE_IBSS) ath10k_warn(ar, "no tim ie found;\n"); return; } tim = (void *)ie + 2; ie_len = ie[1]; pvm_len = ie_len - 3; /* exclude dtim count, dtim period, bmap ctl */ if (pvm_len < arvif->u.ap.tim_len) { int expand_size = tim_len - pvm_len; int move_size = skb_tail_pointer(bcn) - (ie + 2 + ie_len); void *next_ie = ie + 2 + ie_len; if (skb_put(bcn, expand_size)) { memmove(next_ie + expand_size, next_ie, move_size); ie[1] += expand_size; ie_len += expand_size; pvm_len += expand_size; } else { ath10k_warn(ar, "tim expansion failed\n"); } } if (pvm_len > tim_len) { ath10k_warn(ar, "tim pvm length is too great (%d)\n", pvm_len); return; } tim->bitmap_ctrl = !!__le32_to_cpu(tim_info->tim_mcast); memcpy(tim->virtual_map, arvif->u.ap.tim_bitmap, pvm_len); if (tim->dtim_count == 0) { ATH10K_SKB_CB(bcn)->flags |= ATH10K_SKB_F_DTIM_ZERO; if (__le32_to_cpu(tim_info->tim_mcast) == 1) ATH10K_SKB_CB(bcn)->flags |= ATH10K_SKB_F_DELIVER_CAB; } ath10k_dbg(ar, ATH10K_DBG_MGMT, "dtim %d/%d mcast %d pvmlen %d\n", tim->dtim_count, tim->dtim_period, tim->bitmap_ctrl, pvm_len); } static void ath10k_wmi_update_noa(struct ath10k *ar, struct ath10k_vif *arvif, struct sk_buff *bcn, const struct wmi_p2p_noa_info *noa) { if (!arvif->vif->p2p) return; ath10k_dbg(ar, ATH10K_DBG_MGMT, "noa changed: %d\n", noa->changed); if (noa->changed & WMI_P2P_NOA_CHANGED_BIT) ath10k_p2p_noa_update(arvif, noa); if (arvif->u.ap.noa_data) if (!pskb_expand_head(bcn, 0, arvif->u.ap.noa_len, GFP_ATOMIC)) skb_put_data(bcn, arvif->u.ap.noa_data, arvif->u.ap.noa_len); } static int ath10k_wmi_op_pull_swba_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_swba_ev_arg *arg) { struct wmi_host_swba_event *ev = (void *)skb->data; u32 map; size_t i; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->vdev_map = ev->vdev_map; for (i = 0, map = __le32_to_cpu(ev->vdev_map); map; map >>= 1) { if (!(map & BIT(0))) continue; /* If this happens there were some changes in firmware and * ath10k should update the max size of tim_info array. */ if (WARN_ON_ONCE(i == ARRAY_SIZE(arg->tim_info))) break; if (__le32_to_cpu(ev->bcn_info[i].tim_info.tim_len) > sizeof(ev->bcn_info[i].tim_info.tim_bitmap)) { ath10k_warn(ar, "refusing to parse invalid swba structure\n"); return -EPROTO; } arg->tim_info[i].tim_len = ev->bcn_info[i].tim_info.tim_len; arg->tim_info[i].tim_mcast = ev->bcn_info[i].tim_info.tim_mcast; arg->tim_info[i].tim_bitmap = ev->bcn_info[i].tim_info.tim_bitmap; arg->tim_info[i].tim_changed = ev->bcn_info[i].tim_info.tim_changed; arg->tim_info[i].tim_num_ps_pending = ev->bcn_info[i].tim_info.tim_num_ps_pending; arg->noa_info[i] = &ev->bcn_info[i].p2p_noa_info; i++; } return 0; } static int ath10k_wmi_10_2_4_op_pull_swba_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_swba_ev_arg *arg) { struct wmi_10_2_4_host_swba_event *ev = (void *)skb->data; u32 map; size_t i; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->vdev_map = ev->vdev_map; for (i = 0, map = __le32_to_cpu(ev->vdev_map); map; map >>= 1) { if (!(map & BIT(0))) continue; /* If this happens there were some changes in firmware and * ath10k should update the max size of tim_info array. */ if (WARN_ON_ONCE(i == ARRAY_SIZE(arg->tim_info))) break; if (__le32_to_cpu(ev->bcn_info[i].tim_info.tim_len) > sizeof(ev->bcn_info[i].tim_info.tim_bitmap)) { ath10k_warn(ar, "refusing to parse invalid swba structure\n"); return -EPROTO; } arg->tim_info[i].tim_len = ev->bcn_info[i].tim_info.tim_len; arg->tim_info[i].tim_mcast = ev->bcn_info[i].tim_info.tim_mcast; arg->tim_info[i].tim_bitmap = ev->bcn_info[i].tim_info.tim_bitmap; arg->tim_info[i].tim_changed = ev->bcn_info[i].tim_info.tim_changed; arg->tim_info[i].tim_num_ps_pending = ev->bcn_info[i].tim_info.tim_num_ps_pending; i++; } return 0; } static int ath10k_wmi_10_4_op_pull_swba_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_swba_ev_arg *arg) { struct wmi_10_4_host_swba_event *ev = (void *)skb->data; u32 map, tim_len; size_t i; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->vdev_map = ev->vdev_map; for (i = 0, map = __le32_to_cpu(ev->vdev_map); map; map >>= 1) { if (!(map & BIT(0))) continue; /* If this happens there were some changes in firmware and * ath10k should update the max size of tim_info array. */ if (WARN_ON_ONCE(i == ARRAY_SIZE(arg->tim_info))) break; if (__le32_to_cpu(ev->bcn_info[i].tim_info.tim_len) > sizeof(ev->bcn_info[i].tim_info.tim_bitmap)) { ath10k_warn(ar, "refusing to parse invalid swba structure\n"); return -EPROTO; } tim_len = __le32_to_cpu(ev->bcn_info[i].tim_info.tim_len); if (tim_len) { /* Exclude 4 byte guard length */ tim_len -= 4; arg->tim_info[i].tim_len = __cpu_to_le32(tim_len); } else { arg->tim_info[i].tim_len = 0; } arg->tim_info[i].tim_mcast = ev->bcn_info[i].tim_info.tim_mcast; arg->tim_info[i].tim_bitmap = ev->bcn_info[i].tim_info.tim_bitmap; arg->tim_info[i].tim_changed = ev->bcn_info[i].tim_info.tim_changed; arg->tim_info[i].tim_num_ps_pending = ev->bcn_info[i].tim_info.tim_num_ps_pending; /* 10.4 firmware doesn't have p2p support. notice of absence * info can be ignored for now. */ i++; } return 0; } static enum wmi_txbf_conf ath10k_wmi_10_4_txbf_conf_scheme(struct ath10k *ar) { return WMI_TXBF_CONF_BEFORE_ASSOC; } void ath10k_wmi_event_host_swba(struct ath10k *ar, struct sk_buff *skb) { struct wmi_swba_ev_arg arg = {}; u32 map; int i = -1; const struct wmi_tim_info_arg *tim_info; const struct wmi_p2p_noa_info *noa_info; struct ath10k_vif *arvif; struct sk_buff *bcn; dma_addr_t paddr; int ret, vdev_id = 0; ret = ath10k_wmi_pull_swba(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse swba event: %d\n", ret); return; } map = __le32_to_cpu(arg.vdev_map); ath10k_dbg(ar, ATH10K_DBG_MGMT, "mgmt swba vdev_map 0x%x\n", map); for (; map; map >>= 1, vdev_id++) { if (!(map & 0x1)) continue; i++; if (i >= WMI_MAX_AP_VDEV) { ath10k_warn(ar, "swba has corrupted vdev map\n"); break; } tim_info = &arg.tim_info[i]; noa_info = arg.noa_info[i]; ath10k_dbg(ar, ATH10K_DBG_MGMT, "mgmt event bcn_info %d tim_len %d mcast %d changed %d num_ps_pending %d bitmap 0x%08x%08x%08x%08x\n", i, __le32_to_cpu(tim_info->tim_len), __le32_to_cpu(tim_info->tim_mcast), __le32_to_cpu(tim_info->tim_changed), __le32_to_cpu(tim_info->tim_num_ps_pending), __le32_to_cpu(tim_info->tim_bitmap[3]), __le32_to_cpu(tim_info->tim_bitmap[2]), __le32_to_cpu(tim_info->tim_bitmap[1]), __le32_to_cpu(tim_info->tim_bitmap[0])); /* TODO: Only first 4 word from tim_bitmap is dumped. * Extend debug code to dump full tim_bitmap. */ arvif = ath10k_get_arvif(ar, vdev_id); if (arvif == NULL) { ath10k_warn(ar, "no vif for vdev_id %d found\n", vdev_id); continue; } /* mac80211 would have already asked us to stop beaconing and * bring the vdev down, so continue in that case */ if (!arvif->is_up) continue; /* There are no completions for beacons so wait for next SWBA * before telling mac80211 to decrement CSA counter * * Once CSA counter is completed stop sending beacons until * actual channel switch is done */ if (arvif->vif->bss_conf.csa_active && ieee80211_beacon_cntdwn_is_complete(arvif->vif, 0)) { ieee80211_csa_finish(arvif->vif, 0); continue; } bcn = ieee80211_beacon_get(ar->hw, arvif->vif, 0); if (!bcn) { ath10k_warn(ar, "could not get mac80211 beacon\n"); continue; } ath10k_tx_h_seq_no(arvif->vif, bcn); ath10k_wmi_update_tim(ar, arvif, bcn, tim_info); ath10k_wmi_update_noa(ar, arvif, bcn, noa_info); spin_lock_bh(&ar->data_lock); if (arvif->beacon) { switch (arvif->beacon_state) { case ATH10K_BEACON_SENT: break; case ATH10K_BEACON_SCHEDULED: ath10k_warn(ar, "SWBA overrun on vdev %d, skipped old beacon\n", arvif->vdev_id); break; case ATH10K_BEACON_SENDING: ath10k_warn(ar, "SWBA overrun on vdev %d, skipped new beacon\n", arvif->vdev_id); dev_kfree_skb(bcn); goto skip; } ath10k_mac_vif_beacon_free(arvif); } if (!arvif->beacon_buf) { paddr = dma_map_single(arvif->ar->dev, bcn->data, bcn->len, DMA_TO_DEVICE); ret = dma_mapping_error(arvif->ar->dev, paddr); if (ret) { ath10k_warn(ar, "failed to map beacon: %d\n", ret); dev_kfree_skb_any(bcn); goto skip; } ATH10K_SKB_CB(bcn)->paddr = paddr; } else { if (bcn->len > IEEE80211_MAX_FRAME_LEN) { ath10k_warn(ar, "trimming beacon %d -> %d bytes!\n", bcn->len, IEEE80211_MAX_FRAME_LEN); skb_trim(bcn, IEEE80211_MAX_FRAME_LEN); } memcpy(arvif->beacon_buf, bcn->data, bcn->len); ATH10K_SKB_CB(bcn)->paddr = arvif->beacon_paddr; } arvif->beacon = bcn; arvif->beacon_state = ATH10K_BEACON_SCHEDULED; trace_ath10k_tx_hdr(ar, bcn->data, bcn->len); trace_ath10k_tx_payload(ar, bcn->data, bcn->len); skip: spin_unlock_bh(&ar->data_lock); } ath10k_wmi_tx_beacons_nowait(ar); } void ath10k_wmi_event_tbttoffset_update(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_TBTTOFFSET_UPDATE_EVENTID\n"); } static void ath10k_radar_detected(struct ath10k *ar) { ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs radar detected\n"); ATH10K_DFS_STAT_INC(ar, radar_detected); /* Control radar events reporting in debugfs file * dfs_block_radar_events */ if (ar->dfs_block_radar_events) ath10k_info(ar, "DFS Radar detected, but ignored as requested\n"); else ieee80211_radar_detected(ar->hw, NULL); } static void ath10k_radar_confirmation_work(struct work_struct *work) { struct ath10k *ar = container_of(work, struct ath10k, radar_confirmation_work); struct ath10k_radar_found_info radar_info; int ret, time_left; reinit_completion(&ar->wmi.radar_confirm); spin_lock_bh(&ar->data_lock); memcpy(&radar_info, &ar->last_radar_info, sizeof(radar_info)); spin_unlock_bh(&ar->data_lock); ret = ath10k_wmi_report_radar_found(ar, &radar_info); if (ret) { ath10k_warn(ar, "failed to send radar found %d\n", ret); goto wait_complete; } time_left = wait_for_completion_timeout(&ar->wmi.radar_confirm, ATH10K_WMI_DFS_CONF_TIMEOUT_HZ); if (time_left) { /* DFS Confirmation status event received and * necessary action completed. */ goto wait_complete; } else { /* DFS Confirmation event not received from FW.Considering this * as real radar. */ ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs confirmation not received from fw, considering as radar\n"); goto radar_detected; } radar_detected: ath10k_radar_detected(ar); /* Reset state to allow sending confirmation on consecutive radar * detections, unless radar confirmation is disabled/stopped. */ wait_complete: spin_lock_bh(&ar->data_lock); if (ar->radar_conf_state != ATH10K_RADAR_CONFIRMATION_STOPPED) ar->radar_conf_state = ATH10K_RADAR_CONFIRMATION_IDLE; spin_unlock_bh(&ar->data_lock); } static void ath10k_dfs_radar_report(struct ath10k *ar, struct wmi_phyerr_ev_arg *phyerr, const struct phyerr_radar_report *rr, u64 tsf) { u32 reg0, reg1, tsf32l; struct ieee80211_channel *ch; struct pulse_event pe; struct radar_detector_specs rs; u64 tsf64; u8 rssi, width; struct ath10k_radar_found_info *radar_info; reg0 = __le32_to_cpu(rr->reg0); reg1 = __le32_to_cpu(rr->reg1); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi phyerr radar report chirp %d max_width %d agc_total_gain %d pulse_delta_diff %d\n", MS(reg0, RADAR_REPORT_REG0_PULSE_IS_CHIRP), MS(reg0, RADAR_REPORT_REG0_PULSE_IS_MAX_WIDTH), MS(reg0, RADAR_REPORT_REG0_AGC_TOTAL_GAIN), MS(reg0, RADAR_REPORT_REG0_PULSE_DELTA_DIFF)); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi phyerr radar report pulse_delta_pean %d pulse_sidx %d fft_valid %d agc_mb_gain %d subchan_mask %d\n", MS(reg0, RADAR_REPORT_REG0_PULSE_DELTA_PEAK), MS(reg0, RADAR_REPORT_REG0_PULSE_SIDX), MS(reg1, RADAR_REPORT_REG1_PULSE_SRCH_FFT_VALID), MS(reg1, RADAR_REPORT_REG1_PULSE_AGC_MB_GAIN), MS(reg1, RADAR_REPORT_REG1_PULSE_SUBCHAN_MASK)); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi phyerr radar report pulse_tsf_offset 0x%X pulse_dur: %d\n", MS(reg1, RADAR_REPORT_REG1_PULSE_TSF_OFFSET), MS(reg1, RADAR_REPORT_REG1_PULSE_DUR)); if (!ar->dfs_detector) return; spin_lock_bh(&ar->data_lock); ch = ar->rx_channel; /* fetch target operating channel during channel change */ if (!ch) ch = ar->tgt_oper_chan; spin_unlock_bh(&ar->data_lock); if (!ch) { ath10k_warn(ar, "failed to derive channel for radar pulse, treating as radar\n"); goto radar_detected; } /* report event to DFS pattern detector */ tsf32l = phyerr->tsf_timestamp; tsf64 = tsf & (~0xFFFFFFFFULL); tsf64 |= tsf32l; width = MS(reg1, RADAR_REPORT_REG1_PULSE_DUR); rssi = phyerr->rssi_combined; /* hardware store this as 8 bit signed value, * set to zero if negative number */ if (rssi & 0x80) rssi = 0; pe.ts = tsf64; pe.freq = ch->center_freq; pe.width = width; pe.rssi = rssi; pe.chirp = (MS(reg0, RADAR_REPORT_REG0_PULSE_IS_CHIRP) != 0); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs add pulse freq: %d, width: %d, rssi %d, tsf: %llX\n", pe.freq, pe.width, pe.rssi, pe.ts); ATH10K_DFS_STAT_INC(ar, pulses_detected); if (!ar->dfs_detector->add_pulse(ar->dfs_detector, &pe, &rs)) { ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs no pulse pattern detected, yet\n"); return; } if ((test_bit(WMI_SERVICE_HOST_DFS_CHECK_SUPPORT, ar->wmi.svc_map)) && ar->dfs_detector->region == NL80211_DFS_FCC) { /* Consecutive radar indications need not be * sent to the firmware until we get confirmation * for the previous detected radar. */ spin_lock_bh(&ar->data_lock); if (ar->radar_conf_state != ATH10K_RADAR_CONFIRMATION_IDLE) { spin_unlock_bh(&ar->data_lock); return; } ar->radar_conf_state = ATH10K_RADAR_CONFIRMATION_INPROGRESS; radar_info = &ar->last_radar_info; radar_info->pri_min = rs.pri_min; radar_info->pri_max = rs.pri_max; radar_info->width_min = rs.width_min; radar_info->width_max = rs.width_max; /*TODO Find sidx_min and sidx_max */ radar_info->sidx_min = MS(reg0, RADAR_REPORT_REG0_PULSE_SIDX); radar_info->sidx_max = MS(reg0, RADAR_REPORT_REG0_PULSE_SIDX); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "sending wmi radar found cmd pri_min %d pri_max %d width_min %d width_max %d sidx_min %d sidx_max %d\n", radar_info->pri_min, radar_info->pri_max, radar_info->width_min, radar_info->width_max, radar_info->sidx_min, radar_info->sidx_max); ieee80211_queue_work(ar->hw, &ar->radar_confirmation_work); spin_unlock_bh(&ar->data_lock); return; } radar_detected: ath10k_radar_detected(ar); } static int ath10k_dfs_fft_report(struct ath10k *ar, struct wmi_phyerr_ev_arg *phyerr, const struct phyerr_fft_report *fftr, u64 tsf) { u32 reg0, reg1; u8 rssi, peak_mag; reg0 = __le32_to_cpu(fftr->reg0); reg1 = __le32_to_cpu(fftr->reg1); rssi = phyerr->rssi_combined; ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi phyerr fft report total_gain_db %d base_pwr_db %d fft_chn_idx %d peak_sidx %d\n", MS(reg0, SEARCH_FFT_REPORT_REG0_TOTAL_GAIN_DB), MS(reg0, SEARCH_FFT_REPORT_REG0_BASE_PWR_DB), MS(reg0, SEARCH_FFT_REPORT_REG0_FFT_CHN_IDX), MS(reg0, SEARCH_FFT_REPORT_REG0_PEAK_SIDX)); ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi phyerr fft report rel_pwr_db %d avgpwr_db %d peak_mag %d num_store_bin %d\n", MS(reg1, SEARCH_FFT_REPORT_REG1_RELPWR_DB), MS(reg1, SEARCH_FFT_REPORT_REG1_AVGPWR_DB), MS(reg1, SEARCH_FFT_REPORT_REG1_PEAK_MAG), MS(reg1, SEARCH_FFT_REPORT_REG1_NUM_STR_BINS_IB)); peak_mag = MS(reg1, SEARCH_FFT_REPORT_REG1_PEAK_MAG); /* false event detection */ if (rssi == DFS_RSSI_POSSIBLY_FALSE && peak_mag < 2 * DFS_PEAK_MAG_THOLD_POSSIBLY_FALSE) { ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs false pulse detected\n"); ATH10K_DFS_STAT_INC(ar, pulses_discarded); return -EINVAL; } return 0; } void ath10k_wmi_event_dfs(struct ath10k *ar, struct wmi_phyerr_ev_arg *phyerr, u64 tsf) { int buf_len, tlv_len, res, i = 0; const struct phyerr_tlv *tlv; const struct phyerr_radar_report *rr; const struct phyerr_fft_report *fftr; const u8 *tlv_buf; buf_len = phyerr->buf_len; ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi event dfs err_code %d rssi %d tsfl 0x%X tsf64 0x%llX len %d\n", phyerr->phy_err_code, phyerr->rssi_combined, phyerr->tsf_timestamp, tsf, buf_len); /* Skip event if DFS disabled */ if (!IS_ENABLED(CONFIG_ATH10K_DFS_CERTIFIED)) return; ATH10K_DFS_STAT_INC(ar, pulses_total); while (i < buf_len) { if (i + sizeof(*tlv) > buf_len) { ath10k_warn(ar, "too short buf for tlv header (%d)\n", i); return; } tlv = (struct phyerr_tlv *)&phyerr->buf[i]; tlv_len = __le16_to_cpu(tlv->len); tlv_buf = &phyerr->buf[i + sizeof(*tlv)]; ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "wmi event dfs tlv_len %d tlv_tag 0x%02X tlv_sig 0x%02X\n", tlv_len, tlv->tag, tlv->sig); switch (tlv->tag) { case PHYERR_TLV_TAG_RADAR_PULSE_SUMMARY: if (i + sizeof(*tlv) + sizeof(*rr) > buf_len) { ath10k_warn(ar, "too short radar pulse summary (%d)\n", i); return; } rr = (struct phyerr_radar_report *)tlv_buf; ath10k_dfs_radar_report(ar, phyerr, rr, tsf); break; case PHYERR_TLV_TAG_SEARCH_FFT_REPORT: if (i + sizeof(*tlv) + sizeof(*fftr) > buf_len) { ath10k_warn(ar, "too short fft report (%d)\n", i); return; } fftr = (struct phyerr_fft_report *)tlv_buf; res = ath10k_dfs_fft_report(ar, phyerr, fftr, tsf); if (res) return; break; } i += sizeof(*tlv) + tlv_len; } } void ath10k_wmi_event_spectral_scan(struct ath10k *ar, struct wmi_phyerr_ev_arg *phyerr, u64 tsf) { int buf_len, tlv_len, res, i = 0; struct phyerr_tlv *tlv; const void *tlv_buf; const struct phyerr_fft_report *fftr; size_t fftr_len; buf_len = phyerr->buf_len; while (i < buf_len) { if (i + sizeof(*tlv) > buf_len) { ath10k_warn(ar, "failed to parse phyerr tlv header at byte %d\n", i); return; } tlv = (struct phyerr_tlv *)&phyerr->buf[i]; tlv_len = __le16_to_cpu(tlv->len); tlv_buf = &phyerr->buf[i + sizeof(*tlv)]; if (i + sizeof(*tlv) + tlv_len > buf_len) { ath10k_warn(ar, "failed to parse phyerr tlv payload at byte %d\n", i); return; } switch (tlv->tag) { case PHYERR_TLV_TAG_SEARCH_FFT_REPORT: if (sizeof(*fftr) > tlv_len) { ath10k_warn(ar, "failed to parse fft report at byte %d\n", i); return; } fftr_len = tlv_len - sizeof(*fftr); fftr = tlv_buf; res = ath10k_spectral_process_fft(ar, phyerr, fftr, fftr_len, tsf); if (res < 0) { ath10k_dbg(ar, ATH10K_DBG_WMI, "failed to process fft report: %d\n", res); return; } break; } i += sizeof(*tlv) + tlv_len; } } static int ath10k_wmi_op_pull_phyerr_ev_hdr(struct ath10k *ar, struct sk_buff *skb, struct wmi_phyerr_hdr_arg *arg) { struct wmi_phyerr_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; arg->num_phyerrs = __le32_to_cpu(ev->num_phyerrs); arg->tsf_l32 = __le32_to_cpu(ev->tsf_l32); arg->tsf_u32 = __le32_to_cpu(ev->tsf_u32); arg->buf_len = skb->len - sizeof(*ev); arg->phyerrs = ev->phyerrs; return 0; } static int ath10k_wmi_10_4_op_pull_phyerr_ev_hdr(struct ath10k *ar, struct sk_buff *skb, struct wmi_phyerr_hdr_arg *arg) { struct wmi_10_4_phyerr_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; /* 10.4 firmware always reports only one phyerr */ arg->num_phyerrs = 1; arg->tsf_l32 = __le32_to_cpu(ev->tsf_l32); arg->tsf_u32 = __le32_to_cpu(ev->tsf_u32); arg->buf_len = skb->len; arg->phyerrs = skb->data; return 0; } int ath10k_wmi_op_pull_phyerr_ev(struct ath10k *ar, const void *phyerr_buf, int left_len, struct wmi_phyerr_ev_arg *arg) { const struct wmi_phyerr *phyerr = phyerr_buf; int i; if (left_len < sizeof(*phyerr)) { ath10k_warn(ar, "wrong phyerr event head len %d (need: >=%zd)\n", left_len, sizeof(*phyerr)); return -EINVAL; } arg->tsf_timestamp = __le32_to_cpu(phyerr->tsf_timestamp); arg->freq1 = __le16_to_cpu(phyerr->freq1); arg->freq2 = __le16_to_cpu(phyerr->freq2); arg->rssi_combined = phyerr->rssi_combined; arg->chan_width_mhz = phyerr->chan_width_mhz; arg->buf_len = __le32_to_cpu(phyerr->buf_len); arg->buf = phyerr->buf; arg->hdr_len = sizeof(*phyerr); for (i = 0; i < 4; i++) arg->nf_chains[i] = __le16_to_cpu(phyerr->nf_chains[i]); switch (phyerr->phy_err_code) { case PHY_ERROR_GEN_SPECTRAL_SCAN: arg->phy_err_code = PHY_ERROR_SPECTRAL_SCAN; break; case PHY_ERROR_GEN_FALSE_RADAR_EXT: arg->phy_err_code = PHY_ERROR_FALSE_RADAR_EXT; break; case PHY_ERROR_GEN_RADAR: arg->phy_err_code = PHY_ERROR_RADAR; break; default: arg->phy_err_code = PHY_ERROR_UNKNOWN; break; } return 0; } static int ath10k_wmi_10_4_op_pull_phyerr_ev(struct ath10k *ar, const void *phyerr_buf, int left_len, struct wmi_phyerr_ev_arg *arg) { const struct wmi_10_4_phyerr_event *phyerr = phyerr_buf; u32 phy_err_mask; int i; if (left_len < sizeof(*phyerr)) { ath10k_warn(ar, "wrong phyerr event head len %d (need: >=%zd)\n", left_len, sizeof(*phyerr)); return -EINVAL; } arg->tsf_timestamp = __le32_to_cpu(phyerr->tsf_timestamp); arg->freq1 = __le16_to_cpu(phyerr->freq1); arg->freq2 = __le16_to_cpu(phyerr->freq2); arg->rssi_combined = phyerr->rssi_combined; arg->chan_width_mhz = phyerr->chan_width_mhz; arg->buf_len = __le32_to_cpu(phyerr->buf_len); arg->buf = phyerr->buf; arg->hdr_len = sizeof(*phyerr); for (i = 0; i < 4; i++) arg->nf_chains[i] = __le16_to_cpu(phyerr->nf_chains[i]); phy_err_mask = __le32_to_cpu(phyerr->phy_err_mask[0]); if (phy_err_mask & PHY_ERROR_10_4_SPECTRAL_SCAN_MASK) arg->phy_err_code = PHY_ERROR_SPECTRAL_SCAN; else if (phy_err_mask & PHY_ERROR_10_4_RADAR_MASK) arg->phy_err_code = PHY_ERROR_RADAR; else arg->phy_err_code = PHY_ERROR_UNKNOWN; return 0; } void ath10k_wmi_event_phyerr(struct ath10k *ar, struct sk_buff *skb) { struct wmi_phyerr_hdr_arg hdr_arg = {}; struct wmi_phyerr_ev_arg phyerr_arg = {}; const void *phyerr; u32 count, i, buf_len, phy_err_code; u64 tsf; int left_len, ret; ATH10K_DFS_STAT_INC(ar, phy_errors); ret = ath10k_wmi_pull_phyerr_hdr(ar, skb, &hdr_arg); if (ret) { ath10k_warn(ar, "failed to parse phyerr event hdr: %d\n", ret); return; } /* Check number of included events */ count = hdr_arg.num_phyerrs; left_len = hdr_arg.buf_len; tsf = hdr_arg.tsf_u32; tsf <<= 32; tsf |= hdr_arg.tsf_l32; ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event phyerr count %d tsf64 0x%llX\n", count, tsf); phyerr = hdr_arg.phyerrs; for (i = 0; i < count; i++) { ret = ath10k_wmi_pull_phyerr(ar, phyerr, left_len, &phyerr_arg); if (ret) { ath10k_warn(ar, "failed to parse phyerr event (%d)\n", i); return; } left_len -= phyerr_arg.hdr_len; buf_len = phyerr_arg.buf_len; phy_err_code = phyerr_arg.phy_err_code; if (left_len < buf_len) { ath10k_warn(ar, "single event (%d) wrong buf len\n", i); return; } left_len -= buf_len; switch (phy_err_code) { case PHY_ERROR_RADAR: ath10k_wmi_event_dfs(ar, &phyerr_arg, tsf); break; case PHY_ERROR_SPECTRAL_SCAN: ath10k_wmi_event_spectral_scan(ar, &phyerr_arg, tsf); break; case PHY_ERROR_FALSE_RADAR_EXT: ath10k_wmi_event_dfs(ar, &phyerr_arg, tsf); ath10k_wmi_event_spectral_scan(ar, &phyerr_arg, tsf); break; default: break; } phyerr = phyerr + phyerr_arg.hdr_len + buf_len; } } static int ath10k_wmi_10_4_op_pull_dfs_status_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_dfs_status_ev_arg *arg) { struct wmi_dfs_status_ev_arg *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; arg->status = ev->status; return 0; } static void ath10k_wmi_event_dfs_status_check(struct ath10k *ar, struct sk_buff *skb) { struct wmi_dfs_status_ev_arg status_arg = {}; int ret; ret = ath10k_wmi_pull_dfs_status(ar, skb, &status_arg); if (ret) { ath10k_warn(ar, "failed to parse dfs status event: %d\n", ret); return; } ath10k_dbg(ar, ATH10K_DBG_REGULATORY, "dfs status event received from fw: %d\n", status_arg.status); /* Even in case of radar detection failure we follow the same * behaviour as if radar is detected i.e to switch to a different * channel. */ if (status_arg.status == WMI_HW_RADAR_DETECTED || status_arg.status == WMI_RADAR_DETECTION_FAIL) ath10k_radar_detected(ar); complete(&ar->wmi.radar_confirm); } void ath10k_wmi_event_roam(struct ath10k *ar, struct sk_buff *skb) { struct wmi_roam_ev_arg arg = {}; int ret; u32 vdev_id; u32 reason; s32 rssi; ret = ath10k_wmi_pull_roam_ev(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse roam event: %d\n", ret); return; } vdev_id = __le32_to_cpu(arg.vdev_id); reason = __le32_to_cpu(arg.reason); rssi = __le32_to_cpu(arg.rssi); rssi += WMI_SPECTRAL_NOISE_FLOOR_REF_DEFAULT; ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi roam event vdev %u reason 0x%08x rssi %d\n", vdev_id, reason, rssi); if (reason >= WMI_ROAM_REASON_MAX) ath10k_warn(ar, "ignoring unknown roam event reason %d on vdev %i\n", reason, vdev_id); switch (reason) { case WMI_ROAM_REASON_BEACON_MISS: ath10k_mac_handle_beacon_miss(ar, vdev_id); break; case WMI_ROAM_REASON_BETTER_AP: case WMI_ROAM_REASON_LOW_RSSI: case WMI_ROAM_REASON_SUITABLE_AP_FOUND: case WMI_ROAM_REASON_HO_FAILED: ath10k_warn(ar, "ignoring not implemented roam event reason %d on vdev %i\n", reason, vdev_id); break; } } void ath10k_wmi_event_profile_match(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_PROFILE_MATCH\n"); } void ath10k_wmi_event_debug_print(struct ath10k *ar, struct sk_buff *skb) { char buf[101], c; int i; for (i = 0; i < sizeof(buf) - 1; i++) { if (i >= skb->len) break; c = skb->data[i]; if (c == '\0') break; if (isascii(c) && isprint(c)) buf[i] = c; else buf[i] = '.'; } if (i == sizeof(buf) - 1) ath10k_warn(ar, "wmi debug print truncated: %d\n", skb->len); /* for some reason the debug prints end with \n, remove that */ if (skb->data[i - 1] == '\n') i--; /* the last byte is always reserved for the null character */ buf[i] = '\0'; ath10k_dbg(ar, ATH10K_DBG_WMI_PRINT, "wmi print '%s'\n", buf); } void ath10k_wmi_event_pdev_qvit(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_PDEV_QVIT_EVENTID\n"); } void ath10k_wmi_event_wlan_profile_data(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_WLAN_PROFILE_DATA_EVENTID\n"); } void ath10k_wmi_event_rtt_measurement_report(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_RTT_MEASUREMENT_REPORT_EVENTID\n"); } void ath10k_wmi_event_tsf_measurement_report(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_TSF_MEASUREMENT_REPORT_EVENTID\n"); } void ath10k_wmi_event_rtt_error_report(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_RTT_ERROR_REPORT_EVENTID\n"); } void ath10k_wmi_event_wow_wakeup_host(struct ath10k *ar, struct sk_buff *skb) { struct wmi_wow_ev_arg ev = {}; int ret; complete(&ar->wow.wakeup_completed); ret = ath10k_wmi_pull_wow_event(ar, skb, &ev); if (ret) { ath10k_warn(ar, "failed to parse wow wakeup event: %d\n", ret); return; } ath10k_dbg(ar, ATH10K_DBG_WMI, "wow wakeup host reason %s\n", wow_reason(ev.wake_reason)); } void ath10k_wmi_event_dcs_interference(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_DCS_INTERFERENCE_EVENTID\n"); } static u8 ath10k_tpc_config_get_rate(struct ath10k *ar, struct wmi_pdev_tpc_config_event *ev, u32 rate_idx, u32 num_chains, u32 rate_code, u8 type) { u8 tpc, num_streams, preamble, ch, stm_idx; num_streams = ATH10K_HW_NSS(rate_code); preamble = ATH10K_HW_PREAMBLE(rate_code); ch = num_chains - 1; tpc = min_t(u8, ev->rates_array[rate_idx], ev->max_reg_allow_pow[ch]); if (__le32_to_cpu(ev->num_tx_chain) <= 1) goto out; if (preamble == WMI_RATE_PREAMBLE_CCK) goto out; stm_idx = num_streams - 1; if (num_chains <= num_streams) goto out; switch (type) { case WMI_TPC_TABLE_TYPE_STBC: tpc = min_t(u8, tpc, ev->max_reg_allow_pow_agstbc[ch - 1][stm_idx]); break; case WMI_TPC_TABLE_TYPE_TXBF: tpc = min_t(u8, tpc, ev->max_reg_allow_pow_agtxbf[ch - 1][stm_idx]); break; case WMI_TPC_TABLE_TYPE_CDD: tpc = min_t(u8, tpc, ev->max_reg_allow_pow_agcdd[ch - 1][stm_idx]); break; default: ath10k_warn(ar, "unknown wmi tpc table type: %d\n", type); tpc = 0; break; } out: return tpc; } static void ath10k_tpc_config_disp_tables(struct ath10k *ar, struct wmi_pdev_tpc_config_event *ev, struct ath10k_tpc_stats *tpc_stats, u8 *rate_code, u16 *pream_table, u8 type) { u32 i, j, pream_idx, flags; u8 tpc[WMI_TPC_TX_N_CHAIN]; char tpc_value[WMI_TPC_TX_N_CHAIN * WMI_TPC_BUF_SIZE]; char buff[WMI_TPC_BUF_SIZE]; flags = __le32_to_cpu(ev->flags); switch (type) { case WMI_TPC_TABLE_TYPE_CDD: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_CDD)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "CDD not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; case WMI_TPC_TABLE_TYPE_STBC: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_STBC)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "STBC not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; case WMI_TPC_TABLE_TYPE_TXBF: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_TXBF)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "TXBF not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; default: ath10k_dbg(ar, ATH10K_DBG_WMI, "invalid table type in wmi tpc event: %d\n", type); return; } pream_idx = 0; for (i = 0; i < tpc_stats->rate_max; i++) { memset(tpc_value, 0, sizeof(tpc_value)); memset(buff, 0, sizeof(buff)); if (i == pream_table[pream_idx]) pream_idx++; for (j = 0; j < tpc_stats->num_tx_chain; j++) { tpc[j] = ath10k_tpc_config_get_rate(ar, ev, i, j + 1, rate_code[i], type); snprintf(buff, sizeof(buff), "%8d ", tpc[j]); strlcat(tpc_value, buff, sizeof(tpc_value)); } tpc_stats->tpc_table[type].pream_idx[i] = pream_idx; tpc_stats->tpc_table[type].rate_code[i] = rate_code[i]; memcpy(tpc_stats->tpc_table[type].tpc_value[i], tpc_value, sizeof(tpc_value)); } } void ath10k_wmi_tpc_config_get_rate_code(u8 *rate_code, u16 *pream_table, u32 num_tx_chain) { u32 i, j, pream_idx; u8 rate_idx; /* Create the rate code table based on the chains supported */ rate_idx = 0; pream_idx = 0; /* Fill CCK rate code */ for (i = 0; i < 4; i++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(i, 0, WMI_RATE_PREAMBLE_CCK); rate_idx++; } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill OFDM rate code */ for (i = 0; i < 8; i++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(i, 0, WMI_RATE_PREAMBLE_OFDM); rate_idx++; } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill HT20 rate code */ for (i = 0; i < num_tx_chain; i++) { for (j = 0; j < 8; j++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(j, i, WMI_RATE_PREAMBLE_HT); rate_idx++; } } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill HT40 rate code */ for (i = 0; i < num_tx_chain; i++) { for (j = 0; j < 8; j++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(j, i, WMI_RATE_PREAMBLE_HT); rate_idx++; } } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill VHT20 rate code */ for (i = 0; i < num_tx_chain; i++) { for (j = 0; j < 10; j++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(j, i, WMI_RATE_PREAMBLE_VHT); rate_idx++; } } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill VHT40 rate code */ for (i = 0; i < num_tx_chain; i++) { for (j = 0; j < 10; j++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(j, i, WMI_RATE_PREAMBLE_VHT); rate_idx++; } } pream_table[pream_idx] = rate_idx; pream_idx++; /* Fill VHT80 rate code */ for (i = 0; i < num_tx_chain; i++) { for (j = 0; j < 10; j++) { rate_code[rate_idx] = ATH10K_HW_RATECODE(j, i, WMI_RATE_PREAMBLE_VHT); rate_idx++; } } pream_table[pream_idx] = rate_idx; pream_idx++; rate_code[rate_idx++] = ATH10K_HW_RATECODE(0, 0, WMI_RATE_PREAMBLE_CCK); rate_code[rate_idx++] = ATH10K_HW_RATECODE(0, 0, WMI_RATE_PREAMBLE_OFDM); rate_code[rate_idx++] = ATH10K_HW_RATECODE(0, 0, WMI_RATE_PREAMBLE_CCK); rate_code[rate_idx++] = ATH10K_HW_RATECODE(0, 0, WMI_RATE_PREAMBLE_OFDM); rate_code[rate_idx++] = ATH10K_HW_RATECODE(0, 0, WMI_RATE_PREAMBLE_OFDM); pream_table[pream_idx] = ATH10K_TPC_PREAM_TABLE_END; } void ath10k_wmi_event_pdev_tpc_config(struct ath10k *ar, struct sk_buff *skb) { u32 num_tx_chain, rate_max; u8 rate_code[WMI_TPC_RATE_MAX]; u16 pream_table[WMI_TPC_PREAM_TABLE_MAX]; struct wmi_pdev_tpc_config_event *ev; struct ath10k_tpc_stats *tpc_stats; ev = (struct wmi_pdev_tpc_config_event *)skb->data; num_tx_chain = __le32_to_cpu(ev->num_tx_chain); if (num_tx_chain > WMI_TPC_TX_N_CHAIN) { ath10k_warn(ar, "number of tx chain is %d greater than TPC configured tx chain %d\n", num_tx_chain, WMI_TPC_TX_N_CHAIN); return; } rate_max = __le32_to_cpu(ev->rate_max); if (rate_max > WMI_TPC_RATE_MAX) { ath10k_warn(ar, "number of rate is %d greater than TPC configured rate %d\n", rate_max, WMI_TPC_RATE_MAX); rate_max = WMI_TPC_RATE_MAX; } tpc_stats = kzalloc(sizeof(*tpc_stats), GFP_ATOMIC); if (!tpc_stats) return; ath10k_wmi_tpc_config_get_rate_code(rate_code, pream_table, num_tx_chain); tpc_stats->chan_freq = __le32_to_cpu(ev->chan_freq); tpc_stats->phy_mode = __le32_to_cpu(ev->phy_mode); tpc_stats->ctl = __le32_to_cpu(ev->ctl); tpc_stats->reg_domain = __le32_to_cpu(ev->reg_domain); tpc_stats->twice_antenna_gain = a_sle32_to_cpu(ev->twice_antenna_gain); tpc_stats->twice_antenna_reduction = __le32_to_cpu(ev->twice_antenna_reduction); tpc_stats->power_limit = __le32_to_cpu(ev->power_limit); tpc_stats->twice_max_rd_power = __le32_to_cpu(ev->twice_max_rd_power); tpc_stats->num_tx_chain = num_tx_chain; tpc_stats->rate_max = rate_max; ath10k_tpc_config_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_CDD); ath10k_tpc_config_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_STBC); ath10k_tpc_config_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_TXBF); ath10k_debug_tpc_stats_process(ar, tpc_stats); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event tpc config channel %d mode %d ctl %d regd %d gain %d %d limit %d max_power %d tx_chanins %d rates %d\n", __le32_to_cpu(ev->chan_freq), __le32_to_cpu(ev->phy_mode), __le32_to_cpu(ev->ctl), __le32_to_cpu(ev->reg_domain), a_sle32_to_cpu(ev->twice_antenna_gain), __le32_to_cpu(ev->twice_antenna_reduction), __le32_to_cpu(ev->power_limit), __le32_to_cpu(ev->twice_max_rd_power) / 2, __le32_to_cpu(ev->num_tx_chain), __le32_to_cpu(ev->rate_max)); } static u8 ath10k_wmi_tpc_final_get_rate(struct ath10k *ar, struct wmi_pdev_tpc_final_table_event *ev, u32 rate_idx, u32 num_chains, u32 rate_code, u8 type, u32 pream_idx) { u8 tpc, num_streams, preamble, ch, stm_idx; s8 pow_agcdd, pow_agstbc, pow_agtxbf; int pream; num_streams = ATH10K_HW_NSS(rate_code); preamble = ATH10K_HW_PREAMBLE(rate_code); ch = num_chains - 1; stm_idx = num_streams - 1; pream = -1; if (__le32_to_cpu(ev->chan_freq) <= 2483) { switch (pream_idx) { case WMI_TPC_PREAM_2GHZ_CCK: pream = 0; break; case WMI_TPC_PREAM_2GHZ_OFDM: pream = 1; break; case WMI_TPC_PREAM_2GHZ_HT20: case WMI_TPC_PREAM_2GHZ_VHT20: pream = 2; break; case WMI_TPC_PREAM_2GHZ_HT40: case WMI_TPC_PREAM_2GHZ_VHT40: pream = 3; break; case WMI_TPC_PREAM_2GHZ_VHT80: pream = 4; break; default: pream = -1; break; } } if (__le32_to_cpu(ev->chan_freq) >= 5180) { switch (pream_idx) { case WMI_TPC_PREAM_5GHZ_OFDM: pream = 0; break; case WMI_TPC_PREAM_5GHZ_HT20: case WMI_TPC_PREAM_5GHZ_VHT20: pream = 1; break; case WMI_TPC_PREAM_5GHZ_HT40: case WMI_TPC_PREAM_5GHZ_VHT40: pream = 2; break; case WMI_TPC_PREAM_5GHZ_VHT80: pream = 3; break; case WMI_TPC_PREAM_5GHZ_HTCUP: pream = 4; break; default: pream = -1; break; } } if (pream == -1) { ath10k_warn(ar, "unknown wmi tpc final index and frequency: %u, %u\n", pream_idx, __le32_to_cpu(ev->chan_freq)); tpc = 0; goto out; } if (pream == 4) tpc = min_t(u8, ev->rates_array[rate_idx], ev->max_reg_allow_pow[ch]); else tpc = min_t(u8, min_t(u8, ev->rates_array[rate_idx], ev->max_reg_allow_pow[ch]), ev->ctl_power_table[0][pream][stm_idx]); if (__le32_to_cpu(ev->num_tx_chain) <= 1) goto out; if (preamble == WMI_RATE_PREAMBLE_CCK) goto out; if (num_chains <= num_streams) goto out; switch (type) { case WMI_TPC_TABLE_TYPE_STBC: pow_agstbc = ev->max_reg_allow_pow_agstbc[ch - 1][stm_idx]; if (pream == 4) tpc = min_t(u8, tpc, pow_agstbc); else tpc = min_t(u8, min_t(u8, tpc, pow_agstbc), ev->ctl_power_table[0][pream][stm_idx]); break; case WMI_TPC_TABLE_TYPE_TXBF: pow_agtxbf = ev->max_reg_allow_pow_agtxbf[ch - 1][stm_idx]; if (pream == 4) tpc = min_t(u8, tpc, pow_agtxbf); else tpc = min_t(u8, min_t(u8, tpc, pow_agtxbf), ev->ctl_power_table[1][pream][stm_idx]); break; case WMI_TPC_TABLE_TYPE_CDD: pow_agcdd = ev->max_reg_allow_pow_agcdd[ch - 1][stm_idx]; if (pream == 4) tpc = min_t(u8, tpc, pow_agcdd); else tpc = min_t(u8, min_t(u8, tpc, pow_agcdd), ev->ctl_power_table[0][pream][stm_idx]); break; default: ath10k_warn(ar, "unknown wmi tpc final table type: %d\n", type); tpc = 0; break; } out: return tpc; } static void ath10k_wmi_tpc_stats_final_disp_tables(struct ath10k *ar, struct wmi_pdev_tpc_final_table_event *ev, struct ath10k_tpc_stats_final *tpc_stats, u8 *rate_code, u16 *pream_table, u8 type) { u32 i, j, pream_idx, flags; u8 tpc[WMI_TPC_TX_N_CHAIN]; char tpc_value[WMI_TPC_TX_N_CHAIN * WMI_TPC_BUF_SIZE]; char buff[WMI_TPC_BUF_SIZE]; flags = __le32_to_cpu(ev->flags); switch (type) { case WMI_TPC_TABLE_TYPE_CDD: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_CDD)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "CDD not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; case WMI_TPC_TABLE_TYPE_STBC: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_STBC)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "STBC not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; case WMI_TPC_TABLE_TYPE_TXBF: if (!(flags & WMI_TPC_CONFIG_EVENT_FLAG_TABLE_TXBF)) { ath10k_dbg(ar, ATH10K_DBG_WMI, "TXBF not supported\n"); tpc_stats->flag[type] = ATH10K_TPC_TABLE_TYPE_FLAG; return; } break; default: ath10k_dbg(ar, ATH10K_DBG_WMI, "invalid table type in wmi tpc event: %d\n", type); return; } pream_idx = 0; for (i = 0; i < tpc_stats->rate_max; i++) { memset(tpc_value, 0, sizeof(tpc_value)); memset(buff, 0, sizeof(buff)); if (i == pream_table[pream_idx]) pream_idx++; for (j = 0; j < tpc_stats->num_tx_chain; j++) { tpc[j] = ath10k_wmi_tpc_final_get_rate(ar, ev, i, j + 1, rate_code[i], type, pream_idx); snprintf(buff, sizeof(buff), "%8d ", tpc[j]); strlcat(tpc_value, buff, sizeof(tpc_value)); } tpc_stats->tpc_table_final[type].pream_idx[i] = pream_idx; tpc_stats->tpc_table_final[type].rate_code[i] = rate_code[i]; memcpy(tpc_stats->tpc_table_final[type].tpc_value[i], tpc_value, sizeof(tpc_value)); } } void ath10k_wmi_event_tpc_final_table(struct ath10k *ar, struct sk_buff *skb) { u32 num_tx_chain, rate_max; u8 rate_code[WMI_TPC_FINAL_RATE_MAX]; u16 pream_table[WMI_TPC_PREAM_TABLE_MAX]; struct wmi_pdev_tpc_final_table_event *ev; struct ath10k_tpc_stats_final *tpc_stats; ev = (struct wmi_pdev_tpc_final_table_event *)skb->data; num_tx_chain = __le32_to_cpu(ev->num_tx_chain); if (num_tx_chain > WMI_TPC_TX_N_CHAIN) { ath10k_warn(ar, "number of tx chain is %d greater than TPC final configured tx chain %d\n", num_tx_chain, WMI_TPC_TX_N_CHAIN); return; } rate_max = __le32_to_cpu(ev->rate_max); if (rate_max > WMI_TPC_FINAL_RATE_MAX) { ath10k_warn(ar, "number of rate is %d greater than TPC final configured rate %d\n", rate_max, WMI_TPC_FINAL_RATE_MAX); rate_max = WMI_TPC_FINAL_RATE_MAX; } tpc_stats = kzalloc(sizeof(*tpc_stats), GFP_ATOMIC); if (!tpc_stats) return; ath10k_wmi_tpc_config_get_rate_code(rate_code, pream_table, num_tx_chain); tpc_stats->chan_freq = __le32_to_cpu(ev->chan_freq); tpc_stats->phy_mode = __le32_to_cpu(ev->phy_mode); tpc_stats->ctl = __le32_to_cpu(ev->ctl); tpc_stats->reg_domain = __le32_to_cpu(ev->reg_domain); tpc_stats->twice_antenna_gain = a_sle32_to_cpu(ev->twice_antenna_gain); tpc_stats->twice_antenna_reduction = __le32_to_cpu(ev->twice_antenna_reduction); tpc_stats->power_limit = __le32_to_cpu(ev->power_limit); tpc_stats->twice_max_rd_power = __le32_to_cpu(ev->twice_max_rd_power); tpc_stats->num_tx_chain = num_tx_chain; tpc_stats->rate_max = rate_max; ath10k_wmi_tpc_stats_final_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_CDD); ath10k_wmi_tpc_stats_final_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_STBC); ath10k_wmi_tpc_stats_final_disp_tables(ar, ev, tpc_stats, rate_code, pream_table, WMI_TPC_TABLE_TYPE_TXBF); ath10k_debug_tpc_stats_final_process(ar, tpc_stats); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event tpc final table channel %d mode %d ctl %d regd %d gain %d %d limit %d max_power %d tx_chanins %d rates %d\n", __le32_to_cpu(ev->chan_freq), __le32_to_cpu(ev->phy_mode), __le32_to_cpu(ev->ctl), __le32_to_cpu(ev->reg_domain), a_sle32_to_cpu(ev->twice_antenna_gain), __le32_to_cpu(ev->twice_antenna_reduction), __le32_to_cpu(ev->power_limit), __le32_to_cpu(ev->twice_max_rd_power) / 2, __le32_to_cpu(ev->num_tx_chain), __le32_to_cpu(ev->rate_max)); } static void ath10k_wmi_handle_tdls_peer_event(struct ath10k *ar, struct sk_buff *skb) { struct wmi_tdls_peer_event *ev; struct ath10k_peer *peer; struct ath10k_vif *arvif; int vdev_id; int peer_status; int peer_reason; u8 reason; if (skb->len < sizeof(*ev)) { ath10k_err(ar, "received tdls peer event with invalid size (%d bytes)\n", skb->len); return; } ev = (struct wmi_tdls_peer_event *)skb->data; vdev_id = __le32_to_cpu(ev->vdev_id); peer_status = __le32_to_cpu(ev->peer_status); peer_reason = __le32_to_cpu(ev->peer_reason); spin_lock_bh(&ar->data_lock); peer = ath10k_peer_find(ar, vdev_id, ev->peer_macaddr.addr); spin_unlock_bh(&ar->data_lock); if (!peer) { ath10k_warn(ar, "failed to find peer entry for %pM\n", ev->peer_macaddr.addr); return; } switch (peer_status) { case WMI_TDLS_SHOULD_TEARDOWN: switch (peer_reason) { case WMI_TDLS_TEARDOWN_REASON_PTR_TIMEOUT: case WMI_TDLS_TEARDOWN_REASON_NO_RESPONSE: case WMI_TDLS_TEARDOWN_REASON_RSSI: reason = WLAN_REASON_TDLS_TEARDOWN_UNREACHABLE; break; default: reason = WLAN_REASON_TDLS_TEARDOWN_UNSPECIFIED; break; } arvif = ath10k_get_arvif(ar, vdev_id); if (!arvif) { ath10k_warn(ar, "received tdls peer event for invalid vdev id %u\n", vdev_id); return; } ieee80211_tdls_oper_request(arvif->vif, ev->peer_macaddr.addr, NL80211_TDLS_TEARDOWN, reason, GFP_ATOMIC); ath10k_dbg(ar, ATH10K_DBG_WMI, "received tdls teardown event for peer %pM reason %u\n", ev->peer_macaddr.addr, peer_reason); break; default: ath10k_dbg(ar, ATH10K_DBG_WMI, "received unknown tdls peer event %u\n", peer_status); break; } } static void ath10k_wmi_event_peer_sta_ps_state_chg(struct ath10k *ar, struct sk_buff *skb) { struct wmi_peer_sta_ps_state_chg_event *ev; struct ieee80211_sta *sta; struct ath10k_sta *arsta; u8 peer_addr[ETH_ALEN]; lockdep_assert_held(&ar->data_lock); ev = (struct wmi_peer_sta_ps_state_chg_event *)skb->data; ether_addr_copy(peer_addr, ev->peer_macaddr.addr); rcu_read_lock(); sta = ieee80211_find_sta_by_ifaddr(ar->hw, peer_addr, NULL); if (!sta) { ath10k_warn(ar, "failed to find station entry %pM\n", peer_addr); goto exit; } arsta = (struct ath10k_sta *)sta->drv_priv; arsta->peer_ps_state = __le32_to_cpu(ev->peer_ps_state); exit: rcu_read_unlock(); } void ath10k_wmi_event_pdev_ftm_intg(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_PDEV_FTM_INTG_EVENTID\n"); } void ath10k_wmi_event_gtk_offload_status(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_GTK_OFFLOAD_STATUS_EVENTID\n"); } void ath10k_wmi_event_gtk_rekey_fail(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_GTK_REKEY_FAIL_EVENTID\n"); } void ath10k_wmi_event_delba_complete(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_TX_DELBA_COMPLETE_EVENTID\n"); } void ath10k_wmi_event_addba_complete(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_TX_ADDBA_COMPLETE_EVENTID\n"); } void ath10k_wmi_event_vdev_install_key_complete(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_VDEV_INSTALL_KEY_COMPLETE_EVENTID\n"); } void ath10k_wmi_event_inst_rssi_stats(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_INST_RSSI_STATS_EVENTID\n"); } void ath10k_wmi_event_vdev_standby_req(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_VDEV_STANDBY_REQ_EVENTID\n"); } void ath10k_wmi_event_vdev_resume_req(struct ath10k *ar, struct sk_buff *skb) { ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI_VDEV_RESUME_REQ_EVENTID\n"); } static int ath10k_wmi_alloc_chunk(struct ath10k *ar, u32 req_id, u32 num_units, u32 unit_len) { dma_addr_t paddr; u32 pool_size; int idx = ar->wmi.num_mem_chunks; void *vaddr; pool_size = num_units * round_up(unit_len, 4); vaddr = dma_alloc_coherent(ar->dev, pool_size, &paddr, GFP_KERNEL); if (!vaddr) return -ENOMEM; ar->wmi.mem_chunks[idx].vaddr = vaddr; ar->wmi.mem_chunks[idx].paddr = paddr; ar->wmi.mem_chunks[idx].len = pool_size; ar->wmi.mem_chunks[idx].req_id = req_id; ar->wmi.num_mem_chunks++; return num_units; } static int ath10k_wmi_alloc_host_mem(struct ath10k *ar, u32 req_id, u32 num_units, u32 unit_len) { int ret; while (num_units) { ret = ath10k_wmi_alloc_chunk(ar, req_id, num_units, unit_len); if (ret < 0) return ret; num_units -= ret; } return 0; } static bool ath10k_wmi_is_host_mem_allocated(struct ath10k *ar, const struct wlan_host_mem_req **mem_reqs, u32 num_mem_reqs) { u32 req_id, num_units, unit_size, num_unit_info; u32 pool_size; int i, j; bool found; if (ar->wmi.num_mem_chunks != num_mem_reqs) return false; for (i = 0; i < num_mem_reqs; ++i) { req_id = __le32_to_cpu(mem_reqs[i]->req_id); num_units = __le32_to_cpu(mem_reqs[i]->num_units); unit_size = __le32_to_cpu(mem_reqs[i]->unit_size); num_unit_info = __le32_to_cpu(mem_reqs[i]->num_unit_info); if (num_unit_info & NUM_UNITS_IS_NUM_ACTIVE_PEERS) { if (ar->num_active_peers) num_units = ar->num_active_peers + 1; else num_units = ar->max_num_peers + 1; } else if (num_unit_info & NUM_UNITS_IS_NUM_PEERS) { num_units = ar->max_num_peers + 1; } else if (num_unit_info & NUM_UNITS_IS_NUM_VDEVS) { num_units = ar->max_num_vdevs + 1; } found = false; for (j = 0; j < ar->wmi.num_mem_chunks; j++) { if (ar->wmi.mem_chunks[j].req_id == req_id) { pool_size = num_units * round_up(unit_size, 4); if (ar->wmi.mem_chunks[j].len == pool_size) { found = true; break; } } } if (!found) return false; } return true; } static int ath10k_wmi_main_op_pull_svc_rdy_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_svc_rdy_ev_arg *arg) { struct wmi_service_ready_event *ev; size_t i, n; if (skb->len < sizeof(*ev)) return -EPROTO; ev = (void *)skb->data; skb_pull(skb, sizeof(*ev)); arg->min_tx_power = ev->hw_min_tx_power; arg->max_tx_power = ev->hw_max_tx_power; arg->ht_cap = ev->ht_cap_info; arg->vht_cap = ev->vht_cap_info; arg->vht_supp_mcs = ev->vht_supp_mcs; arg->sw_ver0 = ev->sw_version; arg->sw_ver1 = ev->sw_version_1; arg->phy_capab = ev->phy_capability; arg->num_rf_chains = ev->num_rf_chains; arg->eeprom_rd = ev->hal_reg_capabilities.eeprom_rd; arg->low_2ghz_chan = ev->hal_reg_capabilities.low_2ghz_chan; arg->high_2ghz_chan = ev->hal_reg_capabilities.high_2ghz_chan; arg->low_5ghz_chan = ev->hal_reg_capabilities.low_5ghz_chan; arg->high_5ghz_chan = ev->hal_reg_capabilities.high_5ghz_chan; arg->num_mem_reqs = ev->num_mem_reqs; arg->service_map = ev->wmi_service_bitmap; arg->service_map_len = sizeof(ev->wmi_service_bitmap); n = min_t(size_t, __le32_to_cpu(arg->num_mem_reqs), ARRAY_SIZE(arg->mem_reqs)); for (i = 0; i < n; i++) arg->mem_reqs[i] = &ev->mem_reqs[i]; if (skb->len < __le32_to_cpu(arg->num_mem_reqs) * sizeof(arg->mem_reqs[0])) return -EPROTO; return 0; } static int ath10k_wmi_10x_op_pull_svc_rdy_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_svc_rdy_ev_arg *arg) { struct wmi_10x_service_ready_event *ev; int i, n; if (skb->len < sizeof(*ev)) return -EPROTO; ev = (void *)skb->data; skb_pull(skb, sizeof(*ev)); arg->min_tx_power = ev->hw_min_tx_power; arg->max_tx_power = ev->hw_max_tx_power; arg->ht_cap = ev->ht_cap_info; arg->vht_cap = ev->vht_cap_info; arg->vht_supp_mcs = ev->vht_supp_mcs; arg->sw_ver0 = ev->sw_version; arg->phy_capab = ev->phy_capability; arg->num_rf_chains = ev->num_rf_chains; arg->eeprom_rd = ev->hal_reg_capabilities.eeprom_rd; arg->low_2ghz_chan = ev->hal_reg_capabilities.low_2ghz_chan; arg->high_2ghz_chan = ev->hal_reg_capabilities.high_2ghz_chan; arg->low_5ghz_chan = ev->hal_reg_capabilities.low_5ghz_chan; arg->high_5ghz_chan = ev->hal_reg_capabilities.high_5ghz_chan; arg->num_mem_reqs = ev->num_mem_reqs; arg->service_map = ev->wmi_service_bitmap; arg->service_map_len = sizeof(ev->wmi_service_bitmap); /* Deliberately skipping ev->sys_cap_info as WMI and WMI-TLV have * different values. We would need a translation to handle that, * but as we don't currently need anything from sys_cap_info from * WMI interface (only from WMI-TLV) safest it to skip it. */ n = min_t(size_t, __le32_to_cpu(arg->num_mem_reqs), ARRAY_SIZE(arg->mem_reqs)); for (i = 0; i < n; i++) arg->mem_reqs[i] = &ev->mem_reqs[i]; if (skb->len < __le32_to_cpu(arg->num_mem_reqs) * sizeof(arg->mem_reqs[0])) return -EPROTO; return 0; } static void ath10k_wmi_event_service_ready_work(struct work_struct *work) { struct ath10k *ar = container_of(work, struct ath10k, svc_rdy_work); struct sk_buff *skb = ar->svc_rdy_skb; struct wmi_svc_rdy_ev_arg arg = {}; u32 num_units, req_id, unit_size, num_mem_reqs, num_unit_info, i; int ret; bool allocated; if (!skb) { ath10k_warn(ar, "invalid service ready event skb\n"); return; } ret = ath10k_wmi_pull_svc_rdy(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse service ready: %d\n", ret); return; } ath10k_wmi_map_svc(ar, arg.service_map, ar->wmi.svc_map, arg.service_map_len); ar->hw_min_tx_power = __le32_to_cpu(arg.min_tx_power); ar->hw_max_tx_power = __le32_to_cpu(arg.max_tx_power); ar->ht_cap_info = __le32_to_cpu(arg.ht_cap); ar->vht_cap_info = __le32_to_cpu(arg.vht_cap); ar->vht_supp_mcs = __le32_to_cpu(arg.vht_supp_mcs); ar->fw_version_major = (__le32_to_cpu(arg.sw_ver0) & 0xff000000) >> 24; ar->fw_version_minor = (__le32_to_cpu(arg.sw_ver0) & 0x00ffffff); ar->fw_version_release = (__le32_to_cpu(arg.sw_ver1) & 0xffff0000) >> 16; ar->fw_version_build = (__le32_to_cpu(arg.sw_ver1) & 0x0000ffff); ar->phy_capability = __le32_to_cpu(arg.phy_capab); ar->num_rf_chains = __le32_to_cpu(arg.num_rf_chains); ar->hw_eeprom_rd = __le32_to_cpu(arg.eeprom_rd); ar->low_2ghz_chan = __le32_to_cpu(arg.low_2ghz_chan); ar->high_2ghz_chan = __le32_to_cpu(arg.high_2ghz_chan); ar->low_5ghz_chan = __le32_to_cpu(arg.low_5ghz_chan); ar->high_5ghz_chan = __le32_to_cpu(arg.high_5ghz_chan); ar->sys_cap_info = __le32_to_cpu(arg.sys_cap_info); ath10k_dbg_dump(ar, ATH10K_DBG_WMI, NULL, "wmi svc: ", arg.service_map, arg.service_map_len); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi sys_cap_info 0x%x\n", ar->sys_cap_info); if (ar->num_rf_chains > ar->max_spatial_stream) { ath10k_warn(ar, "hardware advertises support for more spatial streams than it should (%d > %d)\n", ar->num_rf_chains, ar->max_spatial_stream); ar->num_rf_chains = ar->max_spatial_stream; } if (!ar->cfg_tx_chainmask) { ar->cfg_tx_chainmask = (1 << ar->num_rf_chains) - 1; ar->cfg_rx_chainmask = (1 << ar->num_rf_chains) - 1; } if (strlen(ar->hw->wiphy->fw_version) == 0) { snprintf(ar->hw->wiphy->fw_version, sizeof(ar->hw->wiphy->fw_version), "%u.%u.%u.%u", ar->fw_version_major, ar->fw_version_minor, ar->fw_version_release, ar->fw_version_build); } num_mem_reqs = __le32_to_cpu(arg.num_mem_reqs); if (num_mem_reqs > WMI_MAX_MEM_REQS) { ath10k_warn(ar, "requested memory chunks number (%d) exceeds the limit\n", num_mem_reqs); return; } if (test_bit(WMI_SERVICE_PEER_CACHING, ar->wmi.svc_map)) { if (test_bit(ATH10K_FW_FEATURE_PEER_FLOW_CONTROL, ar->running_fw->fw_file.fw_features)) ar->num_active_peers = TARGET_10_4_QCACHE_ACTIVE_PEERS_PFC + ar->max_num_vdevs; else ar->num_active_peers = TARGET_10_4_QCACHE_ACTIVE_PEERS + ar->max_num_vdevs; ar->max_num_peers = TARGET_10_4_NUM_QCACHE_PEERS_MAX + ar->max_num_vdevs; ar->num_tids = ar->num_active_peers * 2; ar->max_num_stations = TARGET_10_4_NUM_QCACHE_PEERS_MAX; } /* TODO: Adjust max peer count for cases like WMI_SERVICE_RATECTRL_CACHE * and WMI_SERVICE_IRAM_TIDS, etc. */ allocated = ath10k_wmi_is_host_mem_allocated(ar, arg.mem_reqs, num_mem_reqs); if (allocated) goto skip_mem_alloc; /* Either this event is received during boot time or there is a change * in memory requirement from firmware when compared to last request. * Free any old memory and do a fresh allocation based on the current * memory requirement. */ ath10k_wmi_free_host_mem(ar); for (i = 0; i < num_mem_reqs; ++i) { req_id = __le32_to_cpu(arg.mem_reqs[i]->req_id); num_units = __le32_to_cpu(arg.mem_reqs[i]->num_units); unit_size = __le32_to_cpu(arg.mem_reqs[i]->unit_size); num_unit_info = __le32_to_cpu(arg.mem_reqs[i]->num_unit_info); if (num_unit_info & NUM_UNITS_IS_NUM_ACTIVE_PEERS) { if (ar->num_active_peers) num_units = ar->num_active_peers + 1; else num_units = ar->max_num_peers + 1; } else if (num_unit_info & NUM_UNITS_IS_NUM_PEERS) { /* number of units to allocate is number of * peers, 1 extra for self peer on target * this needs to be tied, host and target * can get out of sync */ num_units = ar->max_num_peers + 1; } else if (num_unit_info & NUM_UNITS_IS_NUM_VDEVS) { num_units = ar->max_num_vdevs + 1; } ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi mem_req_id %d num_units %d num_unit_info %d unit size %d actual units %d\n", req_id, __le32_to_cpu(arg.mem_reqs[i]->num_units), num_unit_info, unit_size, num_units); ret = ath10k_wmi_alloc_host_mem(ar, req_id, num_units, unit_size); if (ret) return; } skip_mem_alloc: ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event service ready min_tx_power 0x%08x max_tx_power 0x%08x ht_cap 0x%08x vht_cap 0x%08x vht_supp_mcs 0x%08x sw_ver0 0x%08x sw_ver1 0x%08x fw_build 0x%08x phy_capab 0x%08x num_rf_chains 0x%08x eeprom_rd 0x%08x low_2ghz_chan %d high_2ghz_chan %d low_5ghz_chan %d high_5ghz_chan %d num_mem_reqs 0x%08x\n", __le32_to_cpu(arg.min_tx_power), __le32_to_cpu(arg.max_tx_power), __le32_to_cpu(arg.ht_cap), __le32_to_cpu(arg.vht_cap), __le32_to_cpu(arg.vht_supp_mcs), __le32_to_cpu(arg.sw_ver0), __le32_to_cpu(arg.sw_ver1), __le32_to_cpu(arg.fw_build), __le32_to_cpu(arg.phy_capab), __le32_to_cpu(arg.num_rf_chains), __le32_to_cpu(arg.eeprom_rd), __le32_to_cpu(arg.low_2ghz_chan), __le32_to_cpu(arg.high_2ghz_chan), __le32_to_cpu(arg.low_5ghz_chan), __le32_to_cpu(arg.high_5ghz_chan), __le32_to_cpu(arg.num_mem_reqs)); dev_kfree_skb(skb); ar->svc_rdy_skb = NULL; complete(&ar->wmi.service_ready); } void ath10k_wmi_event_service_ready(struct ath10k *ar, struct sk_buff *skb) { ar->svc_rdy_skb = skb; queue_work(ar->workqueue_aux, &ar->svc_rdy_work); } static int ath10k_wmi_op_pull_rdy_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_rdy_ev_arg *arg) { struct wmi_ready_event *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->sw_version = ev->sw_version; arg->abi_version = ev->abi_version; arg->status = ev->status; arg->mac_addr = ev->mac_addr.addr; return 0; } static int ath10k_wmi_op_pull_roam_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_roam_ev_arg *arg) { struct wmi_roam_ev *ev = (void *)skb->data; if (skb->len < sizeof(*ev)) return -EPROTO; skb_pull(skb, sizeof(*ev)); arg->vdev_id = ev->vdev_id; arg->reason = ev->reason; return 0; } static int ath10k_wmi_op_pull_echo_ev(struct ath10k *ar, struct sk_buff *skb, struct wmi_echo_ev_arg *arg) { struct wmi_echo_event *ev = (void *)skb->data; arg->value = ev->value; return 0; } int ath10k_wmi_event_ready(struct ath10k *ar, struct sk_buff *skb) { struct wmi_rdy_ev_arg arg = {}; int ret; ret = ath10k_wmi_pull_rdy(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse ready event: %d\n", ret); return ret; } ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event ready sw_version 0x%08x abi_version %u mac_addr %pM status %d\n", __le32_to_cpu(arg.sw_version), __le32_to_cpu(arg.abi_version), arg.mac_addr, __le32_to_cpu(arg.status)); if (is_zero_ether_addr(ar->mac_addr)) ether_addr_copy(ar->mac_addr, arg.mac_addr); complete(&ar->wmi.unified_ready); return 0; } void ath10k_wmi_event_service_available(struct ath10k *ar, struct sk_buff *skb) { int ret; struct wmi_svc_avail_ev_arg arg = {}; ret = ath10k_wmi_pull_svc_avail(ar, skb, &arg); if (ret) { ath10k_warn(ar, "failed to parse service available event: %d\n", ret); } /* * Initialization of "arg.service_map_ext_valid" to ZERO is necessary * for the below logic to work. */ if (arg.service_map_ext_valid) ath10k_wmi_map_svc_ext(ar, arg.service_map_ext, ar->wmi.svc_map, __le32_to_cpu(arg.service_map_ext_len)); } static int ath10k_wmi_event_temperature(struct ath10k *ar, struct sk_buff *skb) { const struct wmi_pdev_temperature_event *ev; ev = (struct wmi_pdev_temperature_event *)skb->data; if (WARN_ON(skb->len < sizeof(*ev))) return -EPROTO; ath10k_thermal_event_temperature(ar, __le32_to_cpu(ev->temperature)); return 0; } static int ath10k_wmi_event_pdev_bss_chan_info(struct ath10k *ar, struct sk_buff *skb) { struct wmi_pdev_bss_chan_info_event *ev; struct survey_info *survey; u64 busy, total, tx, rx, rx_bss; u32 freq, noise_floor; u32 cc_freq_hz = ar->hw_params.channel_counters_freq_hz; int idx; ev = (struct wmi_pdev_bss_chan_info_event *)skb->data; if (WARN_ON(skb->len < sizeof(*ev))) return -EPROTO; freq = __le32_to_cpu(ev->freq); noise_floor = __le32_to_cpu(ev->noise_floor); busy = __le64_to_cpu(ev->cycle_busy); total = __le64_to_cpu(ev->cycle_total); tx = __le64_to_cpu(ev->cycle_tx); rx = __le64_to_cpu(ev->cycle_rx); rx_bss = __le64_to_cpu(ev->cycle_rx_bss); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi event pdev bss chan info:\n freq: %d noise: %d cycle: busy %llu total %llu tx %llu rx %llu rx_bss %llu\n", freq, noise_floor, busy, total, tx, rx, rx_bss); spin_lock_bh(&ar->data_lock); idx = freq_to_idx(ar, freq); if (idx >= ARRAY_SIZE(ar->survey)) { ath10k_warn(ar, "bss chan info: invalid frequency %d (idx %d out of bounds)\n", freq, idx); goto exit; } survey = &ar->survey[idx]; survey->noise = noise_floor; survey->time = div_u64(total, cc_freq_hz); survey->time_busy = div_u64(busy, cc_freq_hz); survey->time_rx = div_u64(rx_bss, cc_freq_hz); survey->time_tx = div_u64(tx, cc_freq_hz); survey->filled |= (SURVEY_INFO_NOISE_DBM | SURVEY_INFO_TIME | SURVEY_INFO_TIME_BUSY | SURVEY_INFO_TIME_RX | SURVEY_INFO_TIME_TX); exit: spin_unlock_bh(&ar->data_lock); complete(&ar->bss_survey_done); return 0; } static inline void ath10k_wmi_queue_set_coverage_class_work(struct ath10k *ar) { if (ar->hw_params.hw_ops->set_coverage_class) { spin_lock_bh(&ar->data_lock); /* This call only ensures that the modified coverage class * persists in case the firmware sets the registers back to * their default value. So calling it is only necessary if the * coverage class has a non-zero value. */ if (ar->fw_coverage.coverage_class) queue_work(ar->workqueue, &ar->set_coverage_class_work); spin_unlock_bh(&ar->data_lock); } } static void ath10k_wmi_op_rx(struct ath10k *ar, struct sk_buff *skb) { struct wmi_cmd_hdr *cmd_hdr; enum wmi_event_id id; cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = MS(__le32_to_cpu(cmd_hdr->cmd_id), WMI_CMD_HDR_CMD_ID); if (skb_pull(skb, sizeof(struct wmi_cmd_hdr)) == NULL) goto out; trace_ath10k_wmi_event(ar, id, skb->data, skb->len); switch (id) { case WMI_MGMT_RX_EVENTID: ath10k_wmi_event_mgmt_rx(ar, skb); /* mgmt_rx() owns the skb now! */ return; case WMI_SCAN_EVENTID: ath10k_wmi_event_scan(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_CHAN_INFO_EVENTID: ath10k_wmi_event_chan_info(ar, skb); break; case WMI_ECHO_EVENTID: ath10k_wmi_event_echo(ar, skb); break; case WMI_DEBUG_MESG_EVENTID: ath10k_wmi_event_debug_mesg(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_UPDATE_STATS_EVENTID: ath10k_wmi_event_update_stats(ar, skb); break; case WMI_VDEV_START_RESP_EVENTID: ath10k_wmi_event_vdev_start_resp(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_VDEV_STOPPED_EVENTID: ath10k_wmi_event_vdev_stopped(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_PEER_STA_KICKOUT_EVENTID: ath10k_wmi_event_peer_sta_kickout(ar, skb); break; case WMI_HOST_SWBA_EVENTID: ath10k_wmi_event_host_swba(ar, skb); break; case WMI_TBTTOFFSET_UPDATE_EVENTID: ath10k_wmi_event_tbttoffset_update(ar, skb); break; case WMI_PHYERR_EVENTID: ath10k_wmi_event_phyerr(ar, skb); break; case WMI_ROAM_EVENTID: ath10k_wmi_event_roam(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_PROFILE_MATCH: ath10k_wmi_event_profile_match(ar, skb); break; case WMI_DEBUG_PRINT_EVENTID: ath10k_wmi_event_debug_print(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_PDEV_QVIT_EVENTID: ath10k_wmi_event_pdev_qvit(ar, skb); break; case WMI_WLAN_PROFILE_DATA_EVENTID: ath10k_wmi_event_wlan_profile_data(ar, skb); break; case WMI_RTT_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_rtt_measurement_report(ar, skb); break; case WMI_TSF_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_tsf_measurement_report(ar, skb); break; case WMI_RTT_ERROR_REPORT_EVENTID: ath10k_wmi_event_rtt_error_report(ar, skb); break; case WMI_WOW_WAKEUP_HOST_EVENTID: ath10k_wmi_event_wow_wakeup_host(ar, skb); break; case WMI_DCS_INTERFERENCE_EVENTID: ath10k_wmi_event_dcs_interference(ar, skb); break; case WMI_PDEV_TPC_CONFIG_EVENTID: ath10k_wmi_event_pdev_tpc_config(ar, skb); break; case WMI_PDEV_FTM_INTG_EVENTID: ath10k_wmi_event_pdev_ftm_intg(ar, skb); break; case WMI_GTK_OFFLOAD_STATUS_EVENTID: ath10k_wmi_event_gtk_offload_status(ar, skb); break; case WMI_GTK_REKEY_FAIL_EVENTID: ath10k_wmi_event_gtk_rekey_fail(ar, skb); break; case WMI_TX_DELBA_COMPLETE_EVENTID: ath10k_wmi_event_delba_complete(ar, skb); break; case WMI_TX_ADDBA_COMPLETE_EVENTID: ath10k_wmi_event_addba_complete(ar, skb); break; case WMI_VDEV_INSTALL_KEY_COMPLETE_EVENTID: ath10k_wmi_event_vdev_install_key_complete(ar, skb); break; case WMI_SERVICE_READY_EVENTID: ath10k_wmi_event_service_ready(ar, skb); return; case WMI_READY_EVENTID: ath10k_wmi_event_ready(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_SERVICE_AVAILABLE_EVENTID: ath10k_wmi_event_service_available(ar, skb); break; default: ath10k_warn(ar, "Unknown eventid: %d\n", id); break; } out: dev_kfree_skb(skb); } static void ath10k_wmi_10_1_op_rx(struct ath10k *ar, struct sk_buff *skb) { struct wmi_cmd_hdr *cmd_hdr; enum wmi_10x_event_id id; bool consumed; cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = MS(__le32_to_cpu(cmd_hdr->cmd_id), WMI_CMD_HDR_CMD_ID); if (skb_pull(skb, sizeof(struct wmi_cmd_hdr)) == NULL) goto out; trace_ath10k_wmi_event(ar, id, skb->data, skb->len); consumed = ath10k_tm_event_wmi(ar, id, skb); /* Ready event must be handled normally also in UTF mode so that we * know the UTF firmware has booted, others we are just bypass WMI * events to testmode. */ if (consumed && id != WMI_10X_READY_EVENTID) { ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi testmode consumed 0x%x\n", id); goto out; } switch (id) { case WMI_10X_MGMT_RX_EVENTID: ath10k_wmi_event_mgmt_rx(ar, skb); /* mgmt_rx() owns the skb now! */ return; case WMI_10X_SCAN_EVENTID: ath10k_wmi_event_scan(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_CHAN_INFO_EVENTID: ath10k_wmi_event_chan_info(ar, skb); break; case WMI_10X_ECHO_EVENTID: ath10k_wmi_event_echo(ar, skb); break; case WMI_10X_DEBUG_MESG_EVENTID: ath10k_wmi_event_debug_mesg(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_UPDATE_STATS_EVENTID: ath10k_wmi_event_update_stats(ar, skb); break; case WMI_10X_VDEV_START_RESP_EVENTID: ath10k_wmi_event_vdev_start_resp(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_VDEV_STOPPED_EVENTID: ath10k_wmi_event_vdev_stopped(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_PEER_STA_KICKOUT_EVENTID: ath10k_wmi_event_peer_sta_kickout(ar, skb); break; case WMI_10X_HOST_SWBA_EVENTID: ath10k_wmi_event_host_swba(ar, skb); break; case WMI_10X_TBTTOFFSET_UPDATE_EVENTID: ath10k_wmi_event_tbttoffset_update(ar, skb); break; case WMI_10X_PHYERR_EVENTID: ath10k_wmi_event_phyerr(ar, skb); break; case WMI_10X_ROAM_EVENTID: ath10k_wmi_event_roam(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_PROFILE_MATCH: ath10k_wmi_event_profile_match(ar, skb); break; case WMI_10X_DEBUG_PRINT_EVENTID: ath10k_wmi_event_debug_print(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_PDEV_QVIT_EVENTID: ath10k_wmi_event_pdev_qvit(ar, skb); break; case WMI_10X_WLAN_PROFILE_DATA_EVENTID: ath10k_wmi_event_wlan_profile_data(ar, skb); break; case WMI_10X_RTT_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_rtt_measurement_report(ar, skb); break; case WMI_10X_TSF_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_tsf_measurement_report(ar, skb); break; case WMI_10X_RTT_ERROR_REPORT_EVENTID: ath10k_wmi_event_rtt_error_report(ar, skb); break; case WMI_10X_WOW_WAKEUP_HOST_EVENTID: ath10k_wmi_event_wow_wakeup_host(ar, skb); break; case WMI_10X_DCS_INTERFERENCE_EVENTID: ath10k_wmi_event_dcs_interference(ar, skb); break; case WMI_10X_PDEV_TPC_CONFIG_EVENTID: ath10k_wmi_event_pdev_tpc_config(ar, skb); break; case WMI_10X_INST_RSSI_STATS_EVENTID: ath10k_wmi_event_inst_rssi_stats(ar, skb); break; case WMI_10X_VDEV_STANDBY_REQ_EVENTID: ath10k_wmi_event_vdev_standby_req(ar, skb); break; case WMI_10X_VDEV_RESUME_REQ_EVENTID: ath10k_wmi_event_vdev_resume_req(ar, skb); break; case WMI_10X_SERVICE_READY_EVENTID: ath10k_wmi_event_service_ready(ar, skb); return; case WMI_10X_READY_EVENTID: ath10k_wmi_event_ready(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10X_PDEV_UTF_EVENTID: /* ignore utf events */ break; default: ath10k_warn(ar, "Unknown eventid: %d\n", id); break; } out: dev_kfree_skb(skb); } static void ath10k_wmi_10_2_op_rx(struct ath10k *ar, struct sk_buff *skb) { struct wmi_cmd_hdr *cmd_hdr; enum wmi_10_2_event_id id; bool consumed; cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = MS(__le32_to_cpu(cmd_hdr->cmd_id), WMI_CMD_HDR_CMD_ID); if (skb_pull(skb, sizeof(struct wmi_cmd_hdr)) == NULL) goto out; trace_ath10k_wmi_event(ar, id, skb->data, skb->len); consumed = ath10k_tm_event_wmi(ar, id, skb); /* Ready event must be handled normally also in UTF mode so that we * know the UTF firmware has booted, others we are just bypass WMI * events to testmode. */ if (consumed && id != WMI_10_2_READY_EVENTID) { ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi testmode consumed 0x%x\n", id); goto out; } switch (id) { case WMI_10_2_MGMT_RX_EVENTID: ath10k_wmi_event_mgmt_rx(ar, skb); /* mgmt_rx() owns the skb now! */ return; case WMI_10_2_SCAN_EVENTID: ath10k_wmi_event_scan(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_CHAN_INFO_EVENTID: ath10k_wmi_event_chan_info(ar, skb); break; case WMI_10_2_ECHO_EVENTID: ath10k_wmi_event_echo(ar, skb); break; case WMI_10_2_DEBUG_MESG_EVENTID: ath10k_wmi_event_debug_mesg(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_UPDATE_STATS_EVENTID: ath10k_wmi_event_update_stats(ar, skb); break; case WMI_10_2_VDEV_START_RESP_EVENTID: ath10k_wmi_event_vdev_start_resp(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_VDEV_STOPPED_EVENTID: ath10k_wmi_event_vdev_stopped(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_PEER_STA_KICKOUT_EVENTID: ath10k_wmi_event_peer_sta_kickout(ar, skb); break; case WMI_10_2_HOST_SWBA_EVENTID: ath10k_wmi_event_host_swba(ar, skb); break; case WMI_10_2_TBTTOFFSET_UPDATE_EVENTID: ath10k_wmi_event_tbttoffset_update(ar, skb); break; case WMI_10_2_PHYERR_EVENTID: ath10k_wmi_event_phyerr(ar, skb); break; case WMI_10_2_ROAM_EVENTID: ath10k_wmi_event_roam(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_PROFILE_MATCH: ath10k_wmi_event_profile_match(ar, skb); break; case WMI_10_2_DEBUG_PRINT_EVENTID: ath10k_wmi_event_debug_print(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_PDEV_QVIT_EVENTID: ath10k_wmi_event_pdev_qvit(ar, skb); break; case WMI_10_2_WLAN_PROFILE_DATA_EVENTID: ath10k_wmi_event_wlan_profile_data(ar, skb); break; case WMI_10_2_RTT_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_rtt_measurement_report(ar, skb); break; case WMI_10_2_TSF_MEASUREMENT_REPORT_EVENTID: ath10k_wmi_event_tsf_measurement_report(ar, skb); break; case WMI_10_2_RTT_ERROR_REPORT_EVENTID: ath10k_wmi_event_rtt_error_report(ar, skb); break; case WMI_10_2_WOW_WAKEUP_HOST_EVENTID: ath10k_wmi_event_wow_wakeup_host(ar, skb); break; case WMI_10_2_DCS_INTERFERENCE_EVENTID: ath10k_wmi_event_dcs_interference(ar, skb); break; case WMI_10_2_PDEV_TPC_CONFIG_EVENTID: ath10k_wmi_event_pdev_tpc_config(ar, skb); break; case WMI_10_2_INST_RSSI_STATS_EVENTID: ath10k_wmi_event_inst_rssi_stats(ar, skb); break; case WMI_10_2_VDEV_STANDBY_REQ_EVENTID: ath10k_wmi_event_vdev_standby_req(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_VDEV_RESUME_REQ_EVENTID: ath10k_wmi_event_vdev_resume_req(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_SERVICE_READY_EVENTID: ath10k_wmi_event_service_ready(ar, skb); return; case WMI_10_2_READY_EVENTID: ath10k_wmi_event_ready(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_2_PDEV_TEMPERATURE_EVENTID: ath10k_wmi_event_temperature(ar, skb); break; case WMI_10_2_PDEV_BSS_CHAN_INFO_EVENTID: ath10k_wmi_event_pdev_bss_chan_info(ar, skb); break; case WMI_10_2_RTT_KEEPALIVE_EVENTID: case WMI_10_2_GPIO_INPUT_EVENTID: case WMI_10_2_PEER_RATECODE_LIST_EVENTID: case WMI_10_2_GENERIC_BUFFER_EVENTID: case WMI_10_2_MCAST_BUF_RELEASE_EVENTID: case WMI_10_2_MCAST_LIST_AGEOUT_EVENTID: case WMI_10_2_WDS_PEER_EVENTID: ath10k_dbg(ar, ATH10K_DBG_WMI, "received event id %d not implemented\n", id); break; case WMI_10_2_PEER_STA_PS_STATECHG_EVENTID: ath10k_wmi_event_peer_sta_ps_state_chg(ar, skb); break; default: ath10k_warn(ar, "Unknown eventid: %d\n", id); break; } out: dev_kfree_skb(skb); } static void ath10k_wmi_10_4_op_rx(struct ath10k *ar, struct sk_buff *skb) { struct wmi_cmd_hdr *cmd_hdr; enum wmi_10_4_event_id id; bool consumed; cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = MS(__le32_to_cpu(cmd_hdr->cmd_id), WMI_CMD_HDR_CMD_ID); if (!skb_pull(skb, sizeof(struct wmi_cmd_hdr))) goto out; trace_ath10k_wmi_event(ar, id, skb->data, skb->len); consumed = ath10k_tm_event_wmi(ar, id, skb); /* Ready event must be handled normally also in UTF mode so that we * know the UTF firmware has booted, others we are just bypass WMI * events to testmode. */ if (consumed && id != WMI_10_4_READY_EVENTID) { ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi testmode consumed 0x%x\n", id); goto out; } switch (id) { case WMI_10_4_MGMT_RX_EVENTID: ath10k_wmi_event_mgmt_rx(ar, skb); /* mgmt_rx() owns the skb now! */ return; case WMI_10_4_ECHO_EVENTID: ath10k_wmi_event_echo(ar, skb); break; case WMI_10_4_DEBUG_MESG_EVENTID: ath10k_wmi_event_debug_mesg(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_SERVICE_READY_EVENTID: ath10k_wmi_event_service_ready(ar, skb); return; case WMI_10_4_SCAN_EVENTID: ath10k_wmi_event_scan(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_CHAN_INFO_EVENTID: ath10k_wmi_event_chan_info(ar, skb); break; case WMI_10_4_PHYERR_EVENTID: ath10k_wmi_event_phyerr(ar, skb); break; case WMI_10_4_READY_EVENTID: ath10k_wmi_event_ready(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_PEER_STA_KICKOUT_EVENTID: ath10k_wmi_event_peer_sta_kickout(ar, skb); break; case WMI_10_4_ROAM_EVENTID: ath10k_wmi_event_roam(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_HOST_SWBA_EVENTID: ath10k_wmi_event_host_swba(ar, skb); break; case WMI_10_4_TBTTOFFSET_UPDATE_EVENTID: ath10k_wmi_event_tbttoffset_update(ar, skb); break; case WMI_10_4_DEBUG_PRINT_EVENTID: ath10k_wmi_event_debug_print(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_VDEV_START_RESP_EVENTID: ath10k_wmi_event_vdev_start_resp(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_VDEV_STOPPED_EVENTID: ath10k_wmi_event_vdev_stopped(ar, skb); ath10k_wmi_queue_set_coverage_class_work(ar); break; case WMI_10_4_WOW_WAKEUP_HOST_EVENTID: case WMI_10_4_PEER_RATECODE_LIST_EVENTID: case WMI_10_4_WDS_PEER_EVENTID: case WMI_10_4_DEBUG_FATAL_CONDITION_EVENTID: ath10k_dbg(ar, ATH10K_DBG_WMI, "received event id %d not implemented\n", id); break; case WMI_10_4_UPDATE_STATS_EVENTID: ath10k_wmi_event_update_stats(ar, skb); break; case WMI_10_4_PDEV_TEMPERATURE_EVENTID: ath10k_wmi_event_temperature(ar, skb); break; case WMI_10_4_PDEV_BSS_CHAN_INFO_EVENTID: ath10k_wmi_event_pdev_bss_chan_info(ar, skb); break; case WMI_10_4_PDEV_TPC_CONFIG_EVENTID: ath10k_wmi_event_pdev_tpc_config(ar, skb); break; case WMI_10_4_TDLS_PEER_EVENTID: ath10k_wmi_handle_tdls_peer_event(ar, skb); break; case WMI_10_4_PDEV_TPC_TABLE_EVENTID: ath10k_wmi_event_tpc_final_table(ar, skb); break; case WMI_10_4_DFS_STATUS_CHECK_EVENTID: ath10k_wmi_event_dfs_status_check(ar, skb); break; case WMI_10_4_PEER_STA_PS_STATECHG_EVENTID: ath10k_wmi_event_peer_sta_ps_state_chg(ar, skb); break; default: ath10k_warn(ar, "Unknown eventid: %d\n", id); break; } out: dev_kfree_skb(skb); } static void ath10k_wmi_process_rx(struct ath10k *ar, struct sk_buff *skb) { int ret; ret = ath10k_wmi_rx(ar, skb); if (ret) ath10k_warn(ar, "failed to process wmi rx: %d\n", ret); } int ath10k_wmi_connect(struct ath10k *ar) { int status; struct ath10k_htc_svc_conn_req conn_req; struct ath10k_htc_svc_conn_resp conn_resp; memset(&ar->wmi.svc_map, 0, sizeof(ar->wmi.svc_map)); memset(&conn_req, 0, sizeof(conn_req)); memset(&conn_resp, 0, sizeof(conn_resp)); /* these fields are the same for all service endpoints */ conn_req.ep_ops.ep_tx_complete = ath10k_wmi_htc_tx_complete; conn_req.ep_ops.ep_rx_complete = ath10k_wmi_process_rx; conn_req.ep_ops.ep_tx_credits = ath10k_wmi_op_ep_tx_credits; /* connect to control service */ conn_req.service_id = ATH10K_HTC_SVC_ID_WMI_CONTROL; status = ath10k_htc_connect_service(&ar->htc, &conn_req, &conn_resp); if (status) { ath10k_warn(ar, "failed to connect to WMI CONTROL service status: %d\n", status); return status; } ar->wmi.eid = conn_resp.eid; return 0; } static struct sk_buff * ath10k_wmi_op_gen_pdev_set_base_macaddr(struct ath10k *ar, const u8 macaddr[ETH_ALEN]) { struct wmi_pdev_set_base_macaddr_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_base_macaddr_cmd *)skb->data; ether_addr_copy(cmd->mac_addr.addr, macaddr); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev basemac %pM\n", macaddr); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pdev_set_rd(struct ath10k *ar, u16 rd, u16 rd2g, u16 rd5g, u16 ctl2g, u16 ctl5g, enum wmi_dfs_region dfs_reg) { struct wmi_pdev_set_regdomain_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_regdomain_cmd *)skb->data; cmd->reg_domain = __cpu_to_le32(rd); cmd->reg_domain_2G = __cpu_to_le32(rd2g); cmd->reg_domain_5G = __cpu_to_le32(rd5g); cmd->conformance_test_limit_2G = __cpu_to_le32(ctl2g); cmd->conformance_test_limit_5G = __cpu_to_le32(ctl5g); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev regdomain rd %x rd2g %x rd5g %x ctl2g %x ctl5g %x\n", rd, rd2g, rd5g, ctl2g, ctl5g); return skb; } static struct sk_buff * ath10k_wmi_10x_op_gen_pdev_set_rd(struct ath10k *ar, u16 rd, u16 rd2g, u16 rd5g, u16 ctl2g, u16 ctl5g, enum wmi_dfs_region dfs_reg) { struct wmi_pdev_set_regdomain_cmd_10x *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_regdomain_cmd_10x *)skb->data; cmd->reg_domain = __cpu_to_le32(rd); cmd->reg_domain_2G = __cpu_to_le32(rd2g); cmd->reg_domain_5G = __cpu_to_le32(rd5g); cmd->conformance_test_limit_2G = __cpu_to_le32(ctl2g); cmd->conformance_test_limit_5G = __cpu_to_le32(ctl5g); cmd->dfs_domain = __cpu_to_le32(dfs_reg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev regdomain rd %x rd2g %x rd5g %x ctl2g %x ctl5g %x dfs_region %x\n", rd, rd2g, rd5g, ctl2g, ctl5g, dfs_reg); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pdev_suspend(struct ath10k *ar, u32 suspend_opt) { struct wmi_pdev_suspend_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_suspend_cmd *)skb->data; cmd->suspend_opt = __cpu_to_le32(suspend_opt); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pdev_resume(struct ath10k *ar) { struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, 0); if (!skb) return ERR_PTR(-ENOMEM); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pdev_set_param(struct ath10k *ar, u32 id, u32 value) { struct wmi_pdev_set_param_cmd *cmd; struct sk_buff *skb; if (id == WMI_PDEV_PARAM_UNSUPPORTED) { ath10k_warn(ar, "pdev param %d not supported by firmware\n", id); return ERR_PTR(-EOPNOTSUPP); } skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_param_cmd *)skb->data; cmd->param_id = __cpu_to_le32(id); cmd->param_value = __cpu_to_le32(value); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev set param %d value %d\n", id, value); return skb; } void ath10k_wmi_put_host_mem_chunks(struct ath10k *ar, struct wmi_host_mem_chunks *chunks) { struct host_memory_chunk *chunk; int i; chunks->count = __cpu_to_le32(ar->wmi.num_mem_chunks); for (i = 0; i < ar->wmi.num_mem_chunks; i++) { chunk = &chunks->items[i]; chunk->ptr = __cpu_to_le32(ar->wmi.mem_chunks[i].paddr); chunk->size = __cpu_to_le32(ar->wmi.mem_chunks[i].len); chunk->req_id = __cpu_to_le32(ar->wmi.mem_chunks[i].req_id); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi chunk %d len %d requested, addr 0x%llx\n", i, ar->wmi.mem_chunks[i].len, (unsigned long long)ar->wmi.mem_chunks[i].paddr); } } static struct sk_buff *ath10k_wmi_op_gen_init(struct ath10k *ar) { struct wmi_init_cmd *cmd; struct sk_buff *buf; struct wmi_resource_config config = {}; u32 val; config.num_vdevs = __cpu_to_le32(TARGET_NUM_VDEVS); config.num_peers = __cpu_to_le32(TARGET_NUM_PEERS); config.num_offload_peers = __cpu_to_le32(TARGET_NUM_OFFLOAD_PEERS); config.num_offload_reorder_bufs = __cpu_to_le32(TARGET_NUM_OFFLOAD_REORDER_BUFS); config.num_peer_keys = __cpu_to_le32(TARGET_NUM_PEER_KEYS); config.num_tids = __cpu_to_le32(TARGET_NUM_TIDS); config.ast_skid_limit = __cpu_to_le32(TARGET_AST_SKID_LIMIT); config.tx_chain_mask = __cpu_to_le32(TARGET_TX_CHAIN_MASK); config.rx_chain_mask = __cpu_to_le32(TARGET_RX_CHAIN_MASK); config.rx_timeout_pri_vo = __cpu_to_le32(TARGET_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_vi = __cpu_to_le32(TARGET_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_be = __cpu_to_le32(TARGET_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_bk = __cpu_to_le32(TARGET_RX_TIMEOUT_HI_PRI); config.rx_decap_mode = __cpu_to_le32(ar->wmi.rx_decap_mode); config.scan_max_pending_reqs = __cpu_to_le32(TARGET_SCAN_MAX_PENDING_REQS); config.bmiss_offload_max_vdev = __cpu_to_le32(TARGET_BMISS_OFFLOAD_MAX_VDEV); config.roam_offload_max_vdev = __cpu_to_le32(TARGET_ROAM_OFFLOAD_MAX_VDEV); config.roam_offload_max_ap_profiles = __cpu_to_le32(TARGET_ROAM_OFFLOAD_MAX_AP_PROFILES); config.num_mcast_groups = __cpu_to_le32(TARGET_NUM_MCAST_GROUPS); config.num_mcast_table_elems = __cpu_to_le32(TARGET_NUM_MCAST_TABLE_ELEMS); config.mcast2ucast_mode = __cpu_to_le32(TARGET_MCAST2UCAST_MODE); config.tx_dbg_log_size = __cpu_to_le32(TARGET_TX_DBG_LOG_SIZE); config.num_wds_entries = __cpu_to_le32(TARGET_NUM_WDS_ENTRIES); config.dma_burst_size = __cpu_to_le32(TARGET_DMA_BURST_SIZE); config.mac_aggr_delim = __cpu_to_le32(TARGET_MAC_AGGR_DELIM); val = TARGET_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK; config.rx_skip_defrag_timeout_dup_detection_check = __cpu_to_le32(val); config.vow_config = __cpu_to_le32(TARGET_VOW_CONFIG); config.gtk_offload_max_vdev = __cpu_to_le32(TARGET_GTK_OFFLOAD_MAX_VDEV); config.num_msdu_desc = __cpu_to_le32(TARGET_NUM_MSDU_DESC); config.max_frag_entries = __cpu_to_le32(TARGET_MAX_FRAG_ENTRIES); buf = ath10k_wmi_alloc_skb(ar, struct_size(cmd, mem_chunks.items, ar->wmi.num_mem_chunks)); if (!buf) return ERR_PTR(-ENOMEM); cmd = (struct wmi_init_cmd *)buf->data; memcpy(&cmd->resource_config, &config, sizeof(config)); ath10k_wmi_put_host_mem_chunks(ar, &cmd->mem_chunks); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi init\n"); return buf; } static struct sk_buff *ath10k_wmi_10_1_op_gen_init(struct ath10k *ar) { struct wmi_init_cmd_10x *cmd; struct sk_buff *buf; struct wmi_resource_config_10x config = {}; u32 val; config.num_vdevs = __cpu_to_le32(TARGET_10X_NUM_VDEVS); config.num_peers = __cpu_to_le32(TARGET_10X_NUM_PEERS); config.num_peer_keys = __cpu_to_le32(TARGET_10X_NUM_PEER_KEYS); config.num_tids = __cpu_to_le32(TARGET_10X_NUM_TIDS); config.ast_skid_limit = __cpu_to_le32(TARGET_10X_AST_SKID_LIMIT); config.tx_chain_mask = __cpu_to_le32(TARGET_10X_TX_CHAIN_MASK); config.rx_chain_mask = __cpu_to_le32(TARGET_10X_RX_CHAIN_MASK); config.rx_timeout_pri_vo = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_vi = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_be = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_bk = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_HI_PRI); config.rx_decap_mode = __cpu_to_le32(ar->wmi.rx_decap_mode); config.scan_max_pending_reqs = __cpu_to_le32(TARGET_10X_SCAN_MAX_PENDING_REQS); config.bmiss_offload_max_vdev = __cpu_to_le32(TARGET_10X_BMISS_OFFLOAD_MAX_VDEV); config.roam_offload_max_vdev = __cpu_to_le32(TARGET_10X_ROAM_OFFLOAD_MAX_VDEV); config.roam_offload_max_ap_profiles = __cpu_to_le32(TARGET_10X_ROAM_OFFLOAD_MAX_AP_PROFILES); config.num_mcast_groups = __cpu_to_le32(TARGET_10X_NUM_MCAST_GROUPS); config.num_mcast_table_elems = __cpu_to_le32(TARGET_10X_NUM_MCAST_TABLE_ELEMS); config.mcast2ucast_mode = __cpu_to_le32(TARGET_10X_MCAST2UCAST_MODE); config.tx_dbg_log_size = __cpu_to_le32(TARGET_10X_TX_DBG_LOG_SIZE); config.num_wds_entries = __cpu_to_le32(TARGET_10X_NUM_WDS_ENTRIES); config.dma_burst_size = __cpu_to_le32(TARGET_10X_DMA_BURST_SIZE); config.mac_aggr_delim = __cpu_to_le32(TARGET_10X_MAC_AGGR_DELIM); val = TARGET_10X_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK; config.rx_skip_defrag_timeout_dup_detection_check = __cpu_to_le32(val); config.vow_config = __cpu_to_le32(TARGET_10X_VOW_CONFIG); config.num_msdu_desc = __cpu_to_le32(TARGET_10X_NUM_MSDU_DESC); config.max_frag_entries = __cpu_to_le32(TARGET_10X_MAX_FRAG_ENTRIES); buf = ath10k_wmi_alloc_skb(ar, struct_size(cmd, mem_chunks.items, ar->wmi.num_mem_chunks)); if (!buf) return ERR_PTR(-ENOMEM); cmd = (struct wmi_init_cmd_10x *)buf->data; memcpy(&cmd->resource_config, &config, sizeof(config)); ath10k_wmi_put_host_mem_chunks(ar, &cmd->mem_chunks); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi init 10x\n"); return buf; } static struct sk_buff *ath10k_wmi_10_2_op_gen_init(struct ath10k *ar) { struct wmi_init_cmd_10_2 *cmd; struct sk_buff *buf; struct wmi_resource_config_10x config = {}; u32 val, features; config.num_vdevs = __cpu_to_le32(TARGET_10X_NUM_VDEVS); config.num_peer_keys = __cpu_to_le32(TARGET_10X_NUM_PEER_KEYS); if (ath10k_peer_stats_enabled(ar)) { config.num_peers = __cpu_to_le32(TARGET_10X_TX_STATS_NUM_PEERS); config.num_tids = __cpu_to_le32(TARGET_10X_TX_STATS_NUM_TIDS); } else { config.num_peers = __cpu_to_le32(TARGET_10X_NUM_PEERS); config.num_tids = __cpu_to_le32(TARGET_10X_NUM_TIDS); } config.ast_skid_limit = __cpu_to_le32(TARGET_10X_AST_SKID_LIMIT); config.tx_chain_mask = __cpu_to_le32(TARGET_10X_TX_CHAIN_MASK); config.rx_chain_mask = __cpu_to_le32(TARGET_10X_RX_CHAIN_MASK); config.rx_timeout_pri_vo = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_vi = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_be = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri_bk = __cpu_to_le32(TARGET_10X_RX_TIMEOUT_HI_PRI); config.rx_decap_mode = __cpu_to_le32(ar->wmi.rx_decap_mode); config.scan_max_pending_reqs = __cpu_to_le32(TARGET_10X_SCAN_MAX_PENDING_REQS); config.bmiss_offload_max_vdev = __cpu_to_le32(TARGET_10X_BMISS_OFFLOAD_MAX_VDEV); config.roam_offload_max_vdev = __cpu_to_le32(TARGET_10X_ROAM_OFFLOAD_MAX_VDEV); config.roam_offload_max_ap_profiles = __cpu_to_le32(TARGET_10X_ROAM_OFFLOAD_MAX_AP_PROFILES); config.num_mcast_groups = __cpu_to_le32(TARGET_10X_NUM_MCAST_GROUPS); config.num_mcast_table_elems = __cpu_to_le32(TARGET_10X_NUM_MCAST_TABLE_ELEMS); config.mcast2ucast_mode = __cpu_to_le32(TARGET_10X_MCAST2UCAST_MODE); config.tx_dbg_log_size = __cpu_to_le32(TARGET_10X_TX_DBG_LOG_SIZE); config.num_wds_entries = __cpu_to_le32(TARGET_10X_NUM_WDS_ENTRIES); config.dma_burst_size = __cpu_to_le32(TARGET_10_2_DMA_BURST_SIZE); config.mac_aggr_delim = __cpu_to_le32(TARGET_10X_MAC_AGGR_DELIM); val = TARGET_10X_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK; config.rx_skip_defrag_timeout_dup_detection_check = __cpu_to_le32(val); config.vow_config = __cpu_to_le32(TARGET_10X_VOW_CONFIG); config.num_msdu_desc = __cpu_to_le32(TARGET_10X_NUM_MSDU_DESC); config.max_frag_entries = __cpu_to_le32(TARGET_10X_MAX_FRAG_ENTRIES); buf = ath10k_wmi_alloc_skb(ar, struct_size(cmd, mem_chunks.items, ar->wmi.num_mem_chunks)); if (!buf) return ERR_PTR(-ENOMEM); cmd = (struct wmi_init_cmd_10_2 *)buf->data; features = WMI_10_2_RX_BATCH_MODE; if (test_bit(ATH10K_FLAG_BTCOEX, &ar->dev_flags) && test_bit(WMI_SERVICE_COEX_GPIO, ar->wmi.svc_map)) features |= WMI_10_2_COEX_GPIO; if (ath10k_peer_stats_enabled(ar)) features |= WMI_10_2_PEER_STATS; if (test_bit(WMI_SERVICE_BSS_CHANNEL_INFO_64, ar->wmi.svc_map)) features |= WMI_10_2_BSS_CHAN_INFO; cmd->resource_config.feature_mask = __cpu_to_le32(features); memcpy(&cmd->resource_config.common, &config, sizeof(config)); ath10k_wmi_put_host_mem_chunks(ar, &cmd->mem_chunks); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi init 10.2\n"); return buf; } static struct sk_buff *ath10k_wmi_10_4_op_gen_init(struct ath10k *ar) { struct wmi_init_cmd_10_4 *cmd; struct sk_buff *buf; struct wmi_resource_config_10_4 config = {}; config.num_vdevs = __cpu_to_le32(ar->max_num_vdevs); config.num_peers = __cpu_to_le32(ar->max_num_peers); config.num_active_peers = __cpu_to_le32(ar->num_active_peers); config.num_tids = __cpu_to_le32(ar->num_tids); config.num_offload_peers = __cpu_to_le32(TARGET_10_4_NUM_OFFLOAD_PEERS); config.num_offload_reorder_buffs = __cpu_to_le32(TARGET_10_4_NUM_OFFLOAD_REORDER_BUFFS); config.num_peer_keys = __cpu_to_le32(TARGET_10_4_NUM_PEER_KEYS); config.ast_skid_limit = __cpu_to_le32(TARGET_10_4_AST_SKID_LIMIT); config.tx_chain_mask = __cpu_to_le32(ar->hw_params.tx_chain_mask); config.rx_chain_mask = __cpu_to_le32(ar->hw_params.rx_chain_mask); config.rx_timeout_pri[0] = __cpu_to_le32(TARGET_10_4_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri[1] = __cpu_to_le32(TARGET_10_4_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri[2] = __cpu_to_le32(TARGET_10_4_RX_TIMEOUT_LO_PRI); config.rx_timeout_pri[3] = __cpu_to_le32(TARGET_10_4_RX_TIMEOUT_HI_PRI); config.rx_decap_mode = __cpu_to_le32(ar->wmi.rx_decap_mode); config.scan_max_pending_req = __cpu_to_le32(TARGET_10_4_SCAN_MAX_REQS); config.bmiss_offload_max_vdev = __cpu_to_le32(TARGET_10_4_BMISS_OFFLOAD_MAX_VDEV); config.roam_offload_max_vdev = __cpu_to_le32(TARGET_10_4_ROAM_OFFLOAD_MAX_VDEV); config.roam_offload_max_ap_profiles = __cpu_to_le32(TARGET_10_4_ROAM_OFFLOAD_MAX_PROFILES); config.num_mcast_groups = __cpu_to_le32(TARGET_10_4_NUM_MCAST_GROUPS); config.num_mcast_table_elems = __cpu_to_le32(TARGET_10_4_NUM_MCAST_TABLE_ELEMS); config.mcast2ucast_mode = __cpu_to_le32(TARGET_10_4_MCAST2UCAST_MODE); config.tx_dbg_log_size = __cpu_to_le32(TARGET_10_4_TX_DBG_LOG_SIZE); config.num_wds_entries = __cpu_to_le32(TARGET_10_4_NUM_WDS_ENTRIES); config.dma_burst_size = __cpu_to_le32(TARGET_10_4_DMA_BURST_SIZE); config.mac_aggr_delim = __cpu_to_le32(TARGET_10_4_MAC_AGGR_DELIM); config.rx_skip_defrag_timeout_dup_detection_check = __cpu_to_le32(TARGET_10_4_RX_SKIP_DEFRAG_TIMEOUT_DUP_DETECTION_CHECK); config.vow_config = __cpu_to_le32(TARGET_10_4_VOW_CONFIG); config.gtk_offload_max_vdev = __cpu_to_le32(TARGET_10_4_GTK_OFFLOAD_MAX_VDEV); config.num_msdu_desc = __cpu_to_le32(ar->htt.max_num_pending_tx); config.max_frag_entries = __cpu_to_le32(TARGET_10_4_11AC_TX_MAX_FRAGS); config.max_peer_ext_stats = __cpu_to_le32(TARGET_10_4_MAX_PEER_EXT_STATS); config.smart_ant_cap = __cpu_to_le32(TARGET_10_4_SMART_ANT_CAP); config.bk_minfree = __cpu_to_le32(TARGET_10_4_BK_MIN_FREE); config.be_minfree = __cpu_to_le32(TARGET_10_4_BE_MIN_FREE); config.vi_minfree = __cpu_to_le32(TARGET_10_4_VI_MIN_FREE); config.vo_minfree = __cpu_to_le32(TARGET_10_4_VO_MIN_FREE); config.rx_batchmode = __cpu_to_le32(TARGET_10_4_RX_BATCH_MODE); config.tt_support = __cpu_to_le32(TARGET_10_4_THERMAL_THROTTLING_CONFIG); config.atf_config = __cpu_to_le32(TARGET_10_4_ATF_CONFIG); config.iphdr_pad_config = __cpu_to_le32(TARGET_10_4_IPHDR_PAD_CONFIG); config.qwrap_config = __cpu_to_le32(TARGET_10_4_QWRAP_CONFIG); buf = ath10k_wmi_alloc_skb(ar, struct_size(cmd, mem_chunks.items, ar->wmi.num_mem_chunks)); if (!buf) return ERR_PTR(-ENOMEM); cmd = (struct wmi_init_cmd_10_4 *)buf->data; memcpy(&cmd->resource_config, &config, sizeof(config)); ath10k_wmi_put_host_mem_chunks(ar, &cmd->mem_chunks); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi init 10.4\n"); return buf; } int ath10k_wmi_start_scan_verify(const struct wmi_start_scan_arg *arg) { if (arg->ie_len > WLAN_SCAN_PARAMS_MAX_IE_LEN) return -EINVAL; if (arg->n_channels > ARRAY_SIZE(arg->channels)) return -EINVAL; if (arg->n_ssids > WLAN_SCAN_PARAMS_MAX_SSID) return -EINVAL; if (arg->n_bssids > WLAN_SCAN_PARAMS_MAX_BSSID) return -EINVAL; return 0; } static size_t ath10k_wmi_start_scan_tlvs_len(const struct wmi_start_scan_arg *arg) { int len = 0; if (arg->ie_len) { len += sizeof(struct wmi_ie_data); len += roundup(arg->ie_len, 4); } if (arg->n_channels) { len += sizeof(struct wmi_chan_list); len += sizeof(__le32) * arg->n_channels; } if (arg->n_ssids) { len += sizeof(struct wmi_ssid_list); len += sizeof(struct wmi_ssid) * arg->n_ssids; } if (arg->n_bssids) { len += sizeof(struct wmi_bssid_list); len += sizeof(struct wmi_mac_addr) * arg->n_bssids; } return len; } void ath10k_wmi_put_start_scan_common(struct wmi_start_scan_common *cmn, const struct wmi_start_scan_arg *arg) { u32 scan_id; u32 scan_req_id; scan_id = WMI_HOST_SCAN_REQ_ID_PREFIX; scan_id |= arg->scan_id; scan_req_id = WMI_HOST_SCAN_REQUESTOR_ID_PREFIX; scan_req_id |= arg->scan_req_id; cmn->scan_id = __cpu_to_le32(scan_id); cmn->scan_req_id = __cpu_to_le32(scan_req_id); cmn->vdev_id = __cpu_to_le32(arg->vdev_id); cmn->scan_priority = __cpu_to_le32(arg->scan_priority); cmn->notify_scan_events = __cpu_to_le32(arg->notify_scan_events); cmn->dwell_time_active = __cpu_to_le32(arg->dwell_time_active); cmn->dwell_time_passive = __cpu_to_le32(arg->dwell_time_passive); cmn->min_rest_time = __cpu_to_le32(arg->min_rest_time); cmn->max_rest_time = __cpu_to_le32(arg->max_rest_time); cmn->repeat_probe_time = __cpu_to_le32(arg->repeat_probe_time); cmn->probe_spacing_time = __cpu_to_le32(arg->probe_spacing_time); cmn->idle_time = __cpu_to_le32(arg->idle_time); cmn->max_scan_time = __cpu_to_le32(arg->max_scan_time); cmn->probe_delay = __cpu_to_le32(arg->probe_delay); cmn->scan_ctrl_flags = __cpu_to_le32(arg->scan_ctrl_flags); } static void ath10k_wmi_put_start_scan_tlvs(u8 *tlvs, const struct wmi_start_scan_arg *arg) { struct wmi_ie_data *ie; struct wmi_chan_list *channels; struct wmi_ssid_list *ssids; struct wmi_bssid_list *bssids; void *ptr = tlvs; int i; if (arg->n_channels) { channels = ptr; channels->tag = __cpu_to_le32(WMI_CHAN_LIST_TAG); channels->num_chan = __cpu_to_le32(arg->n_channels); for (i = 0; i < arg->n_channels; i++) channels->channel_list[i].freq = __cpu_to_le16(arg->channels[i]); ptr += sizeof(*channels); ptr += sizeof(__le32) * arg->n_channels; } if (arg->n_ssids) { ssids = ptr; ssids->tag = __cpu_to_le32(WMI_SSID_LIST_TAG); ssids->num_ssids = __cpu_to_le32(arg->n_ssids); for (i = 0; i < arg->n_ssids; i++) { ssids->ssids[i].ssid_len = __cpu_to_le32(arg->ssids[i].len); memcpy(&ssids->ssids[i].ssid, arg->ssids[i].ssid, arg->ssids[i].len); } ptr += sizeof(*ssids); ptr += sizeof(struct wmi_ssid) * arg->n_ssids; } if (arg->n_bssids) { bssids = ptr; bssids->tag = __cpu_to_le32(WMI_BSSID_LIST_TAG); bssids->num_bssid = __cpu_to_le32(arg->n_bssids); for (i = 0; i < arg->n_bssids; i++) ether_addr_copy(bssids->bssid_list[i].addr, arg->bssids[i].bssid); ptr += sizeof(*bssids); ptr += sizeof(struct wmi_mac_addr) * arg->n_bssids; } if (arg->ie_len) { ie = ptr; ie->tag = __cpu_to_le32(WMI_IE_TAG); ie->ie_len = __cpu_to_le32(arg->ie_len); memcpy(ie->ie_data, arg->ie, arg->ie_len); ptr += sizeof(*ie); ptr += roundup(arg->ie_len, 4); } } static struct sk_buff * ath10k_wmi_op_gen_start_scan(struct ath10k *ar, const struct wmi_start_scan_arg *arg) { struct wmi_start_scan_cmd *cmd; struct sk_buff *skb; size_t len; int ret; ret = ath10k_wmi_start_scan_verify(arg); if (ret) return ERR_PTR(ret); len = sizeof(*cmd) + ath10k_wmi_start_scan_tlvs_len(arg); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_start_scan_cmd *)skb->data; ath10k_wmi_put_start_scan_common(&cmd->common, arg); ath10k_wmi_put_start_scan_tlvs(cmd->tlvs, arg); cmd->burst_duration_ms = __cpu_to_le32(0); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi start scan\n"); return skb; } static struct sk_buff * ath10k_wmi_10x_op_gen_start_scan(struct ath10k *ar, const struct wmi_start_scan_arg *arg) { struct wmi_10x_start_scan_cmd *cmd; struct sk_buff *skb; size_t len; int ret; ret = ath10k_wmi_start_scan_verify(arg); if (ret) return ERR_PTR(ret); len = sizeof(*cmd) + ath10k_wmi_start_scan_tlvs_len(arg); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_10x_start_scan_cmd *)skb->data; ath10k_wmi_put_start_scan_common(&cmd->common, arg); ath10k_wmi_put_start_scan_tlvs(cmd->tlvs, arg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi 10x start scan\n"); return skb; } void ath10k_wmi_start_scan_init(struct ath10k *ar, struct wmi_start_scan_arg *arg) { /* setup commonly used values */ arg->scan_req_id = 1; arg->scan_priority = WMI_SCAN_PRIORITY_LOW; arg->dwell_time_active = 50; arg->dwell_time_passive = 150; arg->min_rest_time = 50; arg->max_rest_time = 500; arg->repeat_probe_time = 0; arg->probe_spacing_time = 0; arg->idle_time = 0; arg->max_scan_time = 20000; arg->probe_delay = 5; arg->notify_scan_events = WMI_SCAN_EVENT_STARTED | WMI_SCAN_EVENT_COMPLETED | WMI_SCAN_EVENT_BSS_CHANNEL | WMI_SCAN_EVENT_FOREIGN_CHANNEL | WMI_SCAN_EVENT_FOREIGN_CHANNEL_EXIT | WMI_SCAN_EVENT_DEQUEUED; arg->scan_ctrl_flags |= WMI_SCAN_CHAN_STAT_EVENT; arg->n_bssids = 1; arg->bssids[0].bssid = "\xFF\xFF\xFF\xFF\xFF\xFF"; } static struct sk_buff * ath10k_wmi_op_gen_stop_scan(struct ath10k *ar, const struct wmi_stop_scan_arg *arg) { struct wmi_stop_scan_cmd *cmd; struct sk_buff *skb; u32 scan_id; u32 req_id; if (arg->req_id > 0xFFF) return ERR_PTR(-EINVAL); if (arg->req_type == WMI_SCAN_STOP_ONE && arg->u.scan_id > 0xFFF) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); scan_id = arg->u.scan_id; scan_id |= WMI_HOST_SCAN_REQ_ID_PREFIX; req_id = arg->req_id; req_id |= WMI_HOST_SCAN_REQUESTOR_ID_PREFIX; cmd = (struct wmi_stop_scan_cmd *)skb->data; cmd->req_type = __cpu_to_le32(arg->req_type); cmd->vdev_id = __cpu_to_le32(arg->u.vdev_id); cmd->scan_id = __cpu_to_le32(scan_id); cmd->scan_req_id = __cpu_to_le32(req_id); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi stop scan reqid %d req_type %d vdev/scan_id %d\n", arg->req_id, arg->req_type, arg->u.scan_id); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_create(struct ath10k *ar, u32 vdev_id, enum wmi_vdev_type type, enum wmi_vdev_subtype subtype, const u8 macaddr[ETH_ALEN]) { struct wmi_vdev_create_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_create_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->vdev_type = __cpu_to_le32(type); cmd->vdev_subtype = __cpu_to_le32(subtype); ether_addr_copy(cmd->vdev_macaddr.addr, macaddr); ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI vdev create: id %d type %d subtype %d macaddr %pM\n", vdev_id, type, subtype, macaddr); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_delete(struct ath10k *ar, u32 vdev_id) { struct wmi_vdev_delete_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_delete_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ath10k_dbg(ar, ATH10K_DBG_WMI, "WMI vdev delete id %d\n", vdev_id); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_start(struct ath10k *ar, const struct wmi_vdev_start_request_arg *arg, bool restart) { struct wmi_vdev_start_request_cmd *cmd; struct sk_buff *skb; const char *cmdname; u32 flags = 0; if (WARN_ON(arg->hidden_ssid && !arg->ssid)) return ERR_PTR(-EINVAL); if (WARN_ON(arg->ssid_len > sizeof(cmd->ssid.ssid))) return ERR_PTR(-EINVAL); if (restart) cmdname = "restart"; else cmdname = "start"; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); if (arg->hidden_ssid) flags |= WMI_VDEV_START_HIDDEN_SSID; if (arg->pmf_enabled) flags |= WMI_VDEV_START_PMF_ENABLED; cmd = (struct wmi_vdev_start_request_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(arg->vdev_id); cmd->disable_hw_ack = __cpu_to_le32(arg->disable_hw_ack); cmd->beacon_interval = __cpu_to_le32(arg->bcn_intval); cmd->dtim_period = __cpu_to_le32(arg->dtim_period); cmd->flags = __cpu_to_le32(flags); cmd->bcn_tx_rate = __cpu_to_le32(arg->bcn_tx_rate); cmd->bcn_tx_power = __cpu_to_le32(arg->bcn_tx_power); if (arg->ssid) { cmd->ssid.ssid_len = __cpu_to_le32(arg->ssid_len); memcpy(cmd->ssid.ssid, arg->ssid, arg->ssid_len); } ath10k_wmi_put_wmi_channel(ar, &cmd->chan, &arg->channel); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi vdev %s id 0x%x flags: 0x%0X, freq %d, mode %d, ch_flags: 0x%0X, max_power: %d\n", cmdname, arg->vdev_id, flags, arg->channel.freq, arg->channel.mode, cmd->chan.flags, arg->channel.max_power); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_stop(struct ath10k *ar, u32 vdev_id) { struct wmi_vdev_stop_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_stop_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi vdev stop id 0x%x\n", vdev_id); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_up(struct ath10k *ar, u32 vdev_id, u32 aid, const u8 *bssid) { struct wmi_vdev_up_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_up_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->vdev_assoc_id = __cpu_to_le32(aid); ether_addr_copy(cmd->vdev_bssid.addr, bssid); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi mgmt vdev up id 0x%x assoc id %d bssid %pM\n", vdev_id, aid, bssid); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_down(struct ath10k *ar, u32 vdev_id) { struct wmi_vdev_down_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_down_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi mgmt vdev down id 0x%x\n", vdev_id); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_set_param(struct ath10k *ar, u32 vdev_id, u32 param_id, u32 param_value) { struct wmi_vdev_set_param_cmd *cmd; struct sk_buff *skb; if (param_id == WMI_VDEV_PARAM_UNSUPPORTED) { ath10k_dbg(ar, ATH10K_DBG_WMI, "vdev param %d not supported by firmware\n", param_id); return ERR_PTR(-EOPNOTSUPP); } skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_set_param_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->param_id = __cpu_to_le32(param_id); cmd->param_value = __cpu_to_le32(param_value); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi vdev id 0x%x set param %d value %d\n", vdev_id, param_id, param_value); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_install_key(struct ath10k *ar, const struct wmi_vdev_install_key_arg *arg) { struct wmi_vdev_install_key_cmd *cmd; struct sk_buff *skb; if (arg->key_cipher == WMI_CIPHER_NONE && arg->key_data != NULL) return ERR_PTR(-EINVAL); if (arg->key_cipher != WMI_CIPHER_NONE && arg->key_data == NULL) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd) + arg->key_len); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_install_key_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(arg->vdev_id); cmd->key_idx = __cpu_to_le32(arg->key_idx); cmd->key_flags = __cpu_to_le32(arg->key_flags); cmd->key_cipher = __cpu_to_le32(arg->key_cipher); cmd->key_len = __cpu_to_le32(arg->key_len); cmd->key_txmic_len = __cpu_to_le32(arg->key_txmic_len); cmd->key_rxmic_len = __cpu_to_le32(arg->key_rxmic_len); if (arg->macaddr) ether_addr_copy(cmd->peer_macaddr.addr, arg->macaddr); if (arg->key_data) memcpy(cmd->key_data, arg->key_data, arg->key_len); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi vdev install key idx %d cipher %d len %d\n", arg->key_idx, arg->key_cipher, arg->key_len); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_spectral_conf(struct ath10k *ar, const struct wmi_vdev_spectral_conf_arg *arg) { struct wmi_vdev_spectral_conf_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_spectral_conf_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(arg->vdev_id); cmd->scan_count = __cpu_to_le32(arg->scan_count); cmd->scan_period = __cpu_to_le32(arg->scan_period); cmd->scan_priority = __cpu_to_le32(arg->scan_priority); cmd->scan_fft_size = __cpu_to_le32(arg->scan_fft_size); cmd->scan_gc_ena = __cpu_to_le32(arg->scan_gc_ena); cmd->scan_restart_ena = __cpu_to_le32(arg->scan_restart_ena); cmd->scan_noise_floor_ref = __cpu_to_le32(arg->scan_noise_floor_ref); cmd->scan_init_delay = __cpu_to_le32(arg->scan_init_delay); cmd->scan_nb_tone_thr = __cpu_to_le32(arg->scan_nb_tone_thr); cmd->scan_str_bin_thr = __cpu_to_le32(arg->scan_str_bin_thr); cmd->scan_wb_rpt_mode = __cpu_to_le32(arg->scan_wb_rpt_mode); cmd->scan_rssi_rpt_mode = __cpu_to_le32(arg->scan_rssi_rpt_mode); cmd->scan_rssi_thr = __cpu_to_le32(arg->scan_rssi_thr); cmd->scan_pwr_format = __cpu_to_le32(arg->scan_pwr_format); cmd->scan_rpt_mode = __cpu_to_le32(arg->scan_rpt_mode); cmd->scan_bin_scale = __cpu_to_le32(arg->scan_bin_scale); cmd->scan_dbm_adj = __cpu_to_le32(arg->scan_dbm_adj); cmd->scan_chn_mask = __cpu_to_le32(arg->scan_chn_mask); return skb; } static struct sk_buff * ath10k_wmi_op_gen_vdev_spectral_enable(struct ath10k *ar, u32 vdev_id, u32 trigger, u32 enable) { struct wmi_vdev_spectral_enable_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_vdev_spectral_enable_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->trigger_cmd = __cpu_to_le32(trigger); cmd->enable_cmd = __cpu_to_le32(enable); return skb; } static struct sk_buff * ath10k_wmi_op_gen_peer_create(struct ath10k *ar, u32 vdev_id, const u8 peer_addr[ETH_ALEN], enum wmi_peer_type peer_type) { struct wmi_peer_create_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_peer_create_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, peer_addr); cmd->peer_type = __cpu_to_le32(peer_type); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer create vdev_id %d peer_addr %pM\n", vdev_id, peer_addr); return skb; } static struct sk_buff * ath10k_wmi_op_gen_peer_delete(struct ath10k *ar, u32 vdev_id, const u8 peer_addr[ETH_ALEN]) { struct wmi_peer_delete_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_peer_delete_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, peer_addr); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer delete vdev_id %d peer_addr %pM\n", vdev_id, peer_addr); return skb; } static struct sk_buff * ath10k_wmi_op_gen_peer_flush(struct ath10k *ar, u32 vdev_id, const u8 peer_addr[ETH_ALEN], u32 tid_bitmap) { struct wmi_peer_flush_tids_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_peer_flush_tids_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->peer_tid_bitmap = __cpu_to_le32(tid_bitmap); ether_addr_copy(cmd->peer_macaddr.addr, peer_addr); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer flush vdev_id %d peer_addr %pM tids %08x\n", vdev_id, peer_addr, tid_bitmap); return skb; } static struct sk_buff * ath10k_wmi_op_gen_peer_set_param(struct ath10k *ar, u32 vdev_id, const u8 *peer_addr, enum wmi_peer_param param_id, u32 param_value) { struct wmi_peer_set_param_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_peer_set_param_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->param_id = __cpu_to_le32(param_id); cmd->param_value = __cpu_to_le32(param_value); ether_addr_copy(cmd->peer_macaddr.addr, peer_addr); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi vdev %d peer 0x%pM set param %d value %d\n", vdev_id, peer_addr, param_id, param_value); return skb; } static struct sk_buff *ath10k_wmi_op_gen_gpio_config(struct ath10k *ar, u32 gpio_num, u32 input, u32 pull_type, u32 intr_mode) { struct wmi_gpio_config_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_gpio_config_cmd *)skb->data; cmd->pull_type = __cpu_to_le32(pull_type); cmd->gpio_num = __cpu_to_le32(gpio_num); cmd->input = __cpu_to_le32(input); cmd->intr_mode = __cpu_to_le32(intr_mode); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi gpio_config gpio_num 0x%08x input 0x%08x pull_type 0x%08x intr_mode 0x%08x\n", gpio_num, input, pull_type, intr_mode); return skb; } static struct sk_buff *ath10k_wmi_op_gen_gpio_output(struct ath10k *ar, u32 gpio_num, u32 set) { struct wmi_gpio_output_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_gpio_output_cmd *)skb->data; cmd->gpio_num = __cpu_to_le32(gpio_num); cmd->set = __cpu_to_le32(set); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi gpio_output gpio_num 0x%08x set 0x%08x\n", gpio_num, set); return skb; } static struct sk_buff * ath10k_wmi_op_gen_set_psmode(struct ath10k *ar, u32 vdev_id, enum wmi_sta_ps_mode psmode) { struct wmi_sta_powersave_mode_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_sta_powersave_mode_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->sta_ps_mode = __cpu_to_le32(psmode); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi set powersave id 0x%x mode %d\n", vdev_id, psmode); return skb; } static struct sk_buff * ath10k_wmi_op_gen_set_sta_ps(struct ath10k *ar, u32 vdev_id, enum wmi_sta_powersave_param param_id, u32 value) { struct wmi_sta_powersave_param_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_sta_powersave_param_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->param_id = __cpu_to_le32(param_id); cmd->param_value = __cpu_to_le32(value); ath10k_dbg(ar, ATH10K_DBG_STA, "wmi sta ps param vdev_id 0x%x param %d value %d\n", vdev_id, param_id, value); return skb; } static struct sk_buff * ath10k_wmi_op_gen_set_ap_ps(struct ath10k *ar, u32 vdev_id, const u8 *mac, enum wmi_ap_ps_peer_param param_id, u32 value) { struct wmi_ap_ps_peer_cmd *cmd; struct sk_buff *skb; if (!mac) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_ap_ps_peer_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->param_id = __cpu_to_le32(param_id); cmd->param_value = __cpu_to_le32(value); ether_addr_copy(cmd->peer_macaddr.addr, mac); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi ap ps param vdev_id 0x%X param %d value %d mac_addr %pM\n", vdev_id, param_id, value, mac); return skb; } static struct sk_buff * ath10k_wmi_op_gen_scan_chan_list(struct ath10k *ar, const struct wmi_scan_chan_list_arg *arg) { struct wmi_scan_chan_list_cmd *cmd; struct sk_buff *skb; struct wmi_channel_arg *ch; struct wmi_channel *ci; int i; skb = ath10k_wmi_alloc_skb(ar, struct_size(cmd, chan_info, arg->n_channels)); if (!skb) return ERR_PTR(-EINVAL); cmd = (struct wmi_scan_chan_list_cmd *)skb->data; cmd->num_scan_chans = __cpu_to_le32(arg->n_channels); for (i = 0; i < arg->n_channels; i++) { ch = &arg->channels[i]; ci = &cmd->chan_info[i]; ath10k_wmi_put_wmi_channel(ar, ci, ch); } return skb; } static void ath10k_wmi_peer_assoc_fill(struct ath10k *ar, void *buf, const struct wmi_peer_assoc_complete_arg *arg) { struct wmi_common_peer_assoc_complete_cmd *cmd = buf; cmd->vdev_id = __cpu_to_le32(arg->vdev_id); cmd->peer_new_assoc = __cpu_to_le32(arg->peer_reassoc ? 0 : 1); cmd->peer_associd = __cpu_to_le32(arg->peer_aid); cmd->peer_flags = __cpu_to_le32(arg->peer_flags); cmd->peer_caps = __cpu_to_le32(arg->peer_caps); cmd->peer_listen_intval = __cpu_to_le32(arg->peer_listen_intval); cmd->peer_ht_caps = __cpu_to_le32(arg->peer_ht_caps); cmd->peer_max_mpdu = __cpu_to_le32(arg->peer_max_mpdu); cmd->peer_mpdu_density = __cpu_to_le32(arg->peer_mpdu_density); cmd->peer_rate_caps = __cpu_to_le32(arg->peer_rate_caps); cmd->peer_nss = __cpu_to_le32(arg->peer_num_spatial_streams); cmd->peer_vht_caps = __cpu_to_le32(arg->peer_vht_caps); cmd->peer_phymode = __cpu_to_le32(arg->peer_phymode); ether_addr_copy(cmd->peer_macaddr.addr, arg->addr); cmd->peer_legacy_rates.num_rates = __cpu_to_le32(arg->peer_legacy_rates.num_rates); memcpy(cmd->peer_legacy_rates.rates, arg->peer_legacy_rates.rates, arg->peer_legacy_rates.num_rates); cmd->peer_ht_rates.num_rates = __cpu_to_le32(arg->peer_ht_rates.num_rates); memcpy(cmd->peer_ht_rates.rates, arg->peer_ht_rates.rates, arg->peer_ht_rates.num_rates); cmd->peer_vht_rates.rx_max_rate = __cpu_to_le32(arg->peer_vht_rates.rx_max_rate); cmd->peer_vht_rates.rx_mcs_set = __cpu_to_le32(arg->peer_vht_rates.rx_mcs_set); cmd->peer_vht_rates.tx_max_rate = __cpu_to_le32(arg->peer_vht_rates.tx_max_rate); cmd->peer_vht_rates.tx_mcs_set = __cpu_to_le32(arg->peer_vht_rates.tx_mcs_set); } static void ath10k_wmi_peer_assoc_fill_main(struct ath10k *ar, void *buf, const struct wmi_peer_assoc_complete_arg *arg) { struct wmi_main_peer_assoc_complete_cmd *cmd = buf; ath10k_wmi_peer_assoc_fill(ar, buf, arg); memset(cmd->peer_ht_info, 0, sizeof(cmd->peer_ht_info)); } static void ath10k_wmi_peer_assoc_fill_10_1(struct ath10k *ar, void *buf, const struct wmi_peer_assoc_complete_arg *arg) { ath10k_wmi_peer_assoc_fill(ar, buf, arg); } static void ath10k_wmi_peer_assoc_fill_10_2(struct ath10k *ar, void *buf, const struct wmi_peer_assoc_complete_arg *arg) { struct wmi_10_2_peer_assoc_complete_cmd *cmd = buf; int max_mcs, max_nss; u32 info0; /* TODO: Is using max values okay with firmware? */ max_mcs = 0xf; max_nss = 0xf; info0 = SM(max_mcs, WMI_PEER_ASSOC_INFO0_MAX_MCS_IDX) | SM(max_nss, WMI_PEER_ASSOC_INFO0_MAX_NSS); ath10k_wmi_peer_assoc_fill(ar, buf, arg); cmd->info0 = __cpu_to_le32(info0); } static void ath10k_wmi_peer_assoc_fill_10_4(struct ath10k *ar, void *buf, const struct wmi_peer_assoc_complete_arg *arg) { struct wmi_10_4_peer_assoc_complete_cmd *cmd = buf; ath10k_wmi_peer_assoc_fill_10_2(ar, buf, arg); cmd->peer_bw_rxnss_override = __cpu_to_le32(arg->peer_bw_rxnss_override); } static int ath10k_wmi_peer_assoc_check_arg(const struct wmi_peer_assoc_complete_arg *arg) { if (arg->peer_mpdu_density > 16) return -EINVAL; if (arg->peer_legacy_rates.num_rates > MAX_SUPPORTED_RATES) return -EINVAL; if (arg->peer_ht_rates.num_rates > MAX_SUPPORTED_RATES) return -EINVAL; return 0; } static struct sk_buff * ath10k_wmi_op_gen_peer_assoc(struct ath10k *ar, const struct wmi_peer_assoc_complete_arg *arg) { size_t len = sizeof(struct wmi_main_peer_assoc_complete_cmd); struct sk_buff *skb; int ret; ret = ath10k_wmi_peer_assoc_check_arg(arg); if (ret) return ERR_PTR(ret); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); ath10k_wmi_peer_assoc_fill_main(ar, skb->data, arg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer assoc vdev %d addr %pM (%s)\n", arg->vdev_id, arg->addr, arg->peer_reassoc ? "reassociate" : "new"); return skb; } static struct sk_buff * ath10k_wmi_10_1_op_gen_peer_assoc(struct ath10k *ar, const struct wmi_peer_assoc_complete_arg *arg) { size_t len = sizeof(struct wmi_10_1_peer_assoc_complete_cmd); struct sk_buff *skb; int ret; ret = ath10k_wmi_peer_assoc_check_arg(arg); if (ret) return ERR_PTR(ret); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); ath10k_wmi_peer_assoc_fill_10_1(ar, skb->data, arg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer assoc vdev %d addr %pM (%s)\n", arg->vdev_id, arg->addr, arg->peer_reassoc ? "reassociate" : "new"); return skb; } static struct sk_buff * ath10k_wmi_10_2_op_gen_peer_assoc(struct ath10k *ar, const struct wmi_peer_assoc_complete_arg *arg) { size_t len = sizeof(struct wmi_10_2_peer_assoc_complete_cmd); struct sk_buff *skb; int ret; ret = ath10k_wmi_peer_assoc_check_arg(arg); if (ret) return ERR_PTR(ret); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); ath10k_wmi_peer_assoc_fill_10_2(ar, skb->data, arg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer assoc vdev %d addr %pM (%s)\n", arg->vdev_id, arg->addr, arg->peer_reassoc ? "reassociate" : "new"); return skb; } static struct sk_buff * ath10k_wmi_10_4_op_gen_peer_assoc(struct ath10k *ar, const struct wmi_peer_assoc_complete_arg *arg) { size_t len = sizeof(struct wmi_10_4_peer_assoc_complete_cmd); struct sk_buff *skb; int ret; ret = ath10k_wmi_peer_assoc_check_arg(arg); if (ret) return ERR_PTR(ret); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); ath10k_wmi_peer_assoc_fill_10_4(ar, skb->data, arg); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi peer assoc vdev %d addr %pM (%s)\n", arg->vdev_id, arg->addr, arg->peer_reassoc ? "reassociate" : "new"); return skb; } static struct sk_buff * ath10k_wmi_10_2_op_gen_pdev_get_temperature(struct ath10k *ar) { struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, 0); if (!skb) return ERR_PTR(-ENOMEM); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev get temperature\n"); return skb; } static struct sk_buff * ath10k_wmi_10_2_op_gen_pdev_bss_chan_info(struct ath10k *ar, enum wmi_bss_survey_req_type type) { struct wmi_pdev_chan_info_req_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_chan_info_req_cmd *)skb->data; cmd->type = __cpu_to_le32(type); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev bss info request type %d\n", type); return skb; } /* This function assumes the beacon is already DMA mapped */ static struct sk_buff * ath10k_wmi_op_gen_beacon_dma(struct ath10k *ar, u32 vdev_id, const void *bcn, size_t bcn_len, u32 bcn_paddr, bool dtim_zero, bool deliver_cab) { struct wmi_bcn_tx_ref_cmd *cmd; struct sk_buff *skb; struct ieee80211_hdr *hdr; u16 fc; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); hdr = (struct ieee80211_hdr *)bcn; fc = le16_to_cpu(hdr->frame_control); cmd = (struct wmi_bcn_tx_ref_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->data_len = __cpu_to_le32(bcn_len); cmd->data_ptr = __cpu_to_le32(bcn_paddr); cmd->msdu_id = 0; cmd->frame_control = __cpu_to_le32(fc); cmd->flags = 0; cmd->antenna_mask = __cpu_to_le32(WMI_BCN_TX_REF_DEF_ANTENNA); if (dtim_zero) cmd->flags |= __cpu_to_le32(WMI_BCN_TX_REF_FLAG_DTIM_ZERO); if (deliver_cab) cmd->flags |= __cpu_to_le32(WMI_BCN_TX_REF_FLAG_DELIVER_CAB); return skb; } void ath10k_wmi_set_wmm_param(struct wmi_wmm_params *params, const struct wmi_wmm_params_arg *arg) { params->cwmin = __cpu_to_le32(arg->cwmin); params->cwmax = __cpu_to_le32(arg->cwmax); params->aifs = __cpu_to_le32(arg->aifs); params->txop = __cpu_to_le32(arg->txop); params->acm = __cpu_to_le32(arg->acm); params->no_ack = __cpu_to_le32(arg->no_ack); } static struct sk_buff * ath10k_wmi_op_gen_pdev_set_wmm(struct ath10k *ar, const struct wmi_wmm_params_all_arg *arg) { struct wmi_pdev_set_wmm_params *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_wmm_params *)skb->data; ath10k_wmi_set_wmm_param(&cmd->ac_be, &arg->ac_be); ath10k_wmi_set_wmm_param(&cmd->ac_bk, &arg->ac_bk); ath10k_wmi_set_wmm_param(&cmd->ac_vi, &arg->ac_vi); ath10k_wmi_set_wmm_param(&cmd->ac_vo, &arg->ac_vo); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev set wmm params\n"); return skb; } static struct sk_buff * ath10k_wmi_op_gen_request_stats(struct ath10k *ar, u32 stats_mask) { struct wmi_request_stats_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_request_stats_cmd *)skb->data; cmd->stats_id = __cpu_to_le32(stats_mask); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi request stats 0x%08x\n", stats_mask); return skb; } static struct sk_buff * ath10k_wmi_op_gen_force_fw_hang(struct ath10k *ar, enum wmi_force_fw_hang_type type, u32 delay_ms) { struct wmi_force_fw_hang_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_force_fw_hang_cmd *)skb->data; cmd->type = __cpu_to_le32(type); cmd->delay_ms = __cpu_to_le32(delay_ms); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi force fw hang %d delay %d\n", type, delay_ms); return skb; } static struct sk_buff * ath10k_wmi_op_gen_dbglog_cfg(struct ath10k *ar, u64 module_enable, u32 log_level) { struct wmi_dbglog_cfg_cmd *cmd; struct sk_buff *skb; u32 cfg; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_dbglog_cfg_cmd *)skb->data; if (module_enable) { cfg = SM(log_level, ATH10K_DBGLOG_CFG_LOG_LVL); } else { /* set back defaults, all modules with WARN level */ cfg = SM(ATH10K_DBGLOG_LEVEL_WARN, ATH10K_DBGLOG_CFG_LOG_LVL); module_enable = ~0; } cmd->module_enable = __cpu_to_le32(module_enable); cmd->module_valid = __cpu_to_le32(~0); cmd->config_enable = __cpu_to_le32(cfg); cmd->config_valid = __cpu_to_le32(ATH10K_DBGLOG_CFG_LOG_LVL_MASK); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi dbglog cfg modules %08x %08x config %08x %08x\n", __le32_to_cpu(cmd->module_enable), __le32_to_cpu(cmd->module_valid), __le32_to_cpu(cmd->config_enable), __le32_to_cpu(cmd->config_valid)); return skb; } static struct sk_buff * ath10k_wmi_10_4_op_gen_dbglog_cfg(struct ath10k *ar, u64 module_enable, u32 log_level) { struct wmi_10_4_dbglog_cfg_cmd *cmd; struct sk_buff *skb; u32 cfg; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_10_4_dbglog_cfg_cmd *)skb->data; if (module_enable) { cfg = SM(log_level, ATH10K_DBGLOG_CFG_LOG_LVL); } else { /* set back defaults, all modules with WARN level */ cfg = SM(ATH10K_DBGLOG_LEVEL_WARN, ATH10K_DBGLOG_CFG_LOG_LVL); module_enable = ~0; } cmd->module_enable = __cpu_to_le64(module_enable); cmd->module_valid = __cpu_to_le64(~0); cmd->config_enable = __cpu_to_le32(cfg); cmd->config_valid = __cpu_to_le32(ATH10K_DBGLOG_CFG_LOG_LVL_MASK); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi dbglog cfg modules 0x%016llx 0x%016llx config %08x %08x\n", __le64_to_cpu(cmd->module_enable), __le64_to_cpu(cmd->module_valid), __le32_to_cpu(cmd->config_enable), __le32_to_cpu(cmd->config_valid)); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pktlog_enable(struct ath10k *ar, u32 ev_bitmap) { struct wmi_pdev_pktlog_enable_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); ev_bitmap &= ATH10K_PKTLOG_ANY; cmd = (struct wmi_pdev_pktlog_enable_cmd *)skb->data; cmd->ev_bitmap = __cpu_to_le32(ev_bitmap); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi enable pktlog filter 0x%08x\n", ev_bitmap); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pktlog_disable(struct ath10k *ar) { struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, 0); if (!skb) return ERR_PTR(-ENOMEM); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi disable pktlog\n"); return skb; } static struct sk_buff * ath10k_wmi_op_gen_pdev_set_quiet_mode(struct ath10k *ar, u32 period, u32 duration, u32 next_offset, u32 enabled) { struct wmi_pdev_set_quiet_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_quiet_cmd *)skb->data; cmd->period = __cpu_to_le32(period); cmd->duration = __cpu_to_le32(duration); cmd->next_start = __cpu_to_le32(next_offset); cmd->enabled = __cpu_to_le32(enabled); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi quiet param: period %u duration %u enabled %d\n", period, duration, enabled); return skb; } static struct sk_buff * ath10k_wmi_op_gen_addba_clear_resp(struct ath10k *ar, u32 vdev_id, const u8 *mac) { struct wmi_addba_clear_resp_cmd *cmd; struct sk_buff *skb; if (!mac) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_addba_clear_resp_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, mac); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi addba clear resp vdev_id 0x%X mac_addr %pM\n", vdev_id, mac); return skb; } static struct sk_buff * ath10k_wmi_op_gen_addba_send(struct ath10k *ar, u32 vdev_id, const u8 *mac, u32 tid, u32 buf_size) { struct wmi_addba_send_cmd *cmd; struct sk_buff *skb; if (!mac) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_addba_send_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, mac); cmd->tid = __cpu_to_le32(tid); cmd->buffersize = __cpu_to_le32(buf_size); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi addba send vdev_id 0x%X mac_addr %pM tid %u bufsize %u\n", vdev_id, mac, tid, buf_size); return skb; } static struct sk_buff * ath10k_wmi_op_gen_addba_set_resp(struct ath10k *ar, u32 vdev_id, const u8 *mac, u32 tid, u32 status) { struct wmi_addba_setresponse_cmd *cmd; struct sk_buff *skb; if (!mac) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_addba_setresponse_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, mac); cmd->tid = __cpu_to_le32(tid); cmd->statuscode = __cpu_to_le32(status); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi addba set resp vdev_id 0x%X mac_addr %pM tid %u status %u\n", vdev_id, mac, tid, status); return skb; } static struct sk_buff * ath10k_wmi_op_gen_delba_send(struct ath10k *ar, u32 vdev_id, const u8 *mac, u32 tid, u32 initiator, u32 reason) { struct wmi_delba_send_cmd *cmd; struct sk_buff *skb; if (!mac) return ERR_PTR(-EINVAL); skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_delba_send_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, mac); cmd->tid = __cpu_to_le32(tid); cmd->initiator = __cpu_to_le32(initiator); cmd->reasoncode = __cpu_to_le32(reason); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi delba send vdev_id 0x%X mac_addr %pM tid %u initiator %u reason %u\n", vdev_id, mac, tid, initiator, reason); return skb; } static struct sk_buff * ath10k_wmi_10_2_4_op_gen_pdev_get_tpc_config(struct ath10k *ar, u32 param) { struct wmi_pdev_get_tpc_config_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_get_tpc_config_cmd *)skb->data; cmd->param = __cpu_to_le32(param); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev get tpc config param %d\n", param); return skb; } static void ath10k_wmi_fw_pdev_base_stats_fill(const struct ath10k_fw_stats_pdev *pdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s\n", "ath10k PDEV stats"); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Channel noise floor", pdev->ch_noise_floor); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "Channel TX power", pdev->chan_tx_power); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "TX frame count", pdev->tx_frame_count); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "RX frame count", pdev->rx_frame_count); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "RX clear count", pdev->rx_clear_count); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "Cycle count", pdev->cycle_count); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "PHY error count", pdev->phy_err_count); *length = len; } static void ath10k_wmi_fw_pdev_extra_stats_fill(const struct ath10k_fw_stats_pdev *pdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "RTS bad count", pdev->rts_bad); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "RTS good count", pdev->rts_good); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "FCS bad count", pdev->fcs_bad); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "No beacon count", pdev->no_beacons); len += scnprintf(buf + len, buf_len - len, "%30s %10u\n", "MIB int count", pdev->mib_int_count); len += scnprintf(buf + len, buf_len - len, "\n"); *length = len; } static void ath10k_wmi_fw_pdev_tx_stats_fill(const struct ath10k_fw_stats_pdev *pdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "\n%30s\n", "ath10k PDEV TX stats"); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "HTT cookies queued", pdev->comp_queued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "HTT cookies disp.", pdev->comp_delivered); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MSDU queued", pdev->msdu_enqued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDU queued", pdev->mpdu_enqued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MSDUs dropped", pdev->wmm_drop); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Local enqued", pdev->local_enqued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Local freed", pdev->local_freed); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "HW queued", pdev->hw_queued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PPDUs reaped", pdev->hw_reaped); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Num underruns", pdev->underrun); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PPDUs cleaned", pdev->tx_abort); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs requeued", pdev->mpdus_requeued); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Excessive retries", pdev->tx_ko); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "HW rate", pdev->data_rc); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Sched self triggers", pdev->self_triggers); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Dropped due to SW retries", pdev->sw_retry_failure); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Illegal rate phy errors", pdev->illgl_rate_phy_err); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Pdev continuous xretry", pdev->pdev_cont_xretry); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "TX timeout", pdev->pdev_tx_timeout); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PDEV resets", pdev->pdev_resets); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PHY underrun", pdev->phy_underrun); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDU is more than txop limit", pdev->txop_ovf); *length = len; } static void ath10k_wmi_fw_pdev_rx_stats_fill(const struct ath10k_fw_stats_pdev *pdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "\n%30s\n", "ath10k PDEV RX stats"); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Mid PPDU route change", pdev->mid_ppdu_route_change); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Tot. number of statuses", pdev->status_rcvd); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Extra frags on rings 0", pdev->r0_frags); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Extra frags on rings 1", pdev->r1_frags); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Extra frags on rings 2", pdev->r2_frags); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Extra frags on rings 3", pdev->r3_frags); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MSDUs delivered to HTT", pdev->htt_msdus); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs delivered to HTT", pdev->htt_mpdus); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MSDUs delivered to stack", pdev->loc_msdus); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs delivered to stack", pdev->loc_mpdus); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Oversized AMSDUs", pdev->oversize_amsdu); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PHY errors", pdev->phy_errs); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "PHY errors drops", pdev->phy_err_drop); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDU errors (FCS, MIC, ENC)", pdev->mpdu_errs); *length = len; } static void ath10k_wmi_fw_vdev_stats_fill(const struct ath10k_fw_stats_vdev *vdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; int i; len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "vdev id", vdev->vdev_id); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "beacon snr", vdev->beacon_snr); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "data snr", vdev->data_snr); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num rx frames", vdev->num_rx_frames); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num rts fail", vdev->num_rts_fail); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num rts success", vdev->num_rts_success); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num rx err", vdev->num_rx_err); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num rx discard", vdev->num_rx_discard); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "num tx not acked", vdev->num_tx_not_acked); for (i = 0 ; i < ARRAY_SIZE(vdev->num_tx_frames); i++) len += scnprintf(buf + len, buf_len - len, "%25s [%02d] %u\n", "num tx frames", i, vdev->num_tx_frames[i]); for (i = 0 ; i < ARRAY_SIZE(vdev->num_tx_frames_retries); i++) len += scnprintf(buf + len, buf_len - len, "%25s [%02d] %u\n", "num tx frames retries", i, vdev->num_tx_frames_retries[i]); for (i = 0 ; i < ARRAY_SIZE(vdev->num_tx_frames_failures); i++) len += scnprintf(buf + len, buf_len - len, "%25s [%02d] %u\n", "num tx frames failures", i, vdev->num_tx_frames_failures[i]); for (i = 0 ; i < ARRAY_SIZE(vdev->tx_rate_history); i++) len += scnprintf(buf + len, buf_len - len, "%25s [%02d] 0x%08x\n", "tx rate history", i, vdev->tx_rate_history[i]); for (i = 0 ; i < ARRAY_SIZE(vdev->beacon_rssi_history); i++) len += scnprintf(buf + len, buf_len - len, "%25s [%02d] %u\n", "beacon rssi history", i, vdev->beacon_rssi_history[i]); len += scnprintf(buf + len, buf_len - len, "\n"); *length = len; } static void ath10k_wmi_fw_peer_stats_fill(const struct ath10k_fw_stats_peer *peer, char *buf, u32 *length, bool extended_peer) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "%30s %pM\n", "Peer MAC address", peer->peer_macaddr); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "Peer RSSI", peer->peer_rssi); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "Peer TX rate", peer->peer_tx_rate); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "Peer RX rate", peer->peer_rx_rate); if (!extended_peer) len += scnprintf(buf + len, buf_len - len, "%30s %llu\n", "Peer RX duration", peer->rx_duration); len += scnprintf(buf + len, buf_len - len, "\n"); *length = len; } static void ath10k_wmi_fw_extd_peer_stats_fill(const struct ath10k_fw_extd_stats_peer *peer, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; len += scnprintf(buf + len, buf_len - len, "%30s %pM\n", "Peer MAC address", peer->peer_macaddr); len += scnprintf(buf + len, buf_len - len, "%30s %llu\n", "Peer RX duration", peer->rx_duration); } void ath10k_wmi_main_op_fw_stats_fill(struct ath10k *ar, struct ath10k_fw_stats *fw_stats, char *buf) { u32 len = 0; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; const struct ath10k_fw_stats_pdev *pdev; const struct ath10k_fw_stats_vdev *vdev; const struct ath10k_fw_stats_peer *peer; size_t num_peers; size_t num_vdevs; spin_lock_bh(&ar->data_lock); pdev = list_first_entry_or_null(&fw_stats->pdevs, struct ath10k_fw_stats_pdev, list); if (!pdev) { ath10k_warn(ar, "failed to get pdev stats\n"); goto unlock; } num_peers = list_count_nodes(&fw_stats->peers); num_vdevs = list_count_nodes(&fw_stats->vdevs); ath10k_wmi_fw_pdev_base_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_tx_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_rx_stats_fill(pdev, buf, &len); len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k VDEV stats", num_vdevs); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(vdev, &fw_stats->vdevs, list) { ath10k_wmi_fw_vdev_stats_fill(vdev, buf, &len); } len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k PEER stats", num_peers); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(peer, &fw_stats->peers, list) { ath10k_wmi_fw_peer_stats_fill(peer, buf, &len, fw_stats->extended); } unlock: spin_unlock_bh(&ar->data_lock); if (len >= buf_len) buf[len - 1] = 0; else buf[len] = 0; } void ath10k_wmi_10x_op_fw_stats_fill(struct ath10k *ar, struct ath10k_fw_stats *fw_stats, char *buf) { unsigned int len = 0; unsigned int buf_len = ATH10K_FW_STATS_BUF_SIZE; const struct ath10k_fw_stats_pdev *pdev; const struct ath10k_fw_stats_vdev *vdev; const struct ath10k_fw_stats_peer *peer; size_t num_peers; size_t num_vdevs; spin_lock_bh(&ar->data_lock); pdev = list_first_entry_or_null(&fw_stats->pdevs, struct ath10k_fw_stats_pdev, list); if (!pdev) { ath10k_warn(ar, "failed to get pdev stats\n"); goto unlock; } num_peers = list_count_nodes(&fw_stats->peers); num_vdevs = list_count_nodes(&fw_stats->vdevs); ath10k_wmi_fw_pdev_base_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_extra_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_tx_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_rx_stats_fill(pdev, buf, &len); len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k VDEV stats", num_vdevs); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(vdev, &fw_stats->vdevs, list) { ath10k_wmi_fw_vdev_stats_fill(vdev, buf, &len); } len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k PEER stats", num_peers); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(peer, &fw_stats->peers, list) { ath10k_wmi_fw_peer_stats_fill(peer, buf, &len, fw_stats->extended); } unlock: spin_unlock_bh(&ar->data_lock); if (len >= buf_len) buf[len - 1] = 0; else buf[len] = 0; } static struct sk_buff * ath10k_wmi_op_gen_pdev_enable_adaptive_cca(struct ath10k *ar, u8 enable, u32 detect_level, u32 detect_margin) { struct wmi_pdev_set_adaptive_cca_params *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_set_adaptive_cca_params *)skb->data; cmd->enable = __cpu_to_le32(enable); cmd->cca_detect_level = __cpu_to_le32(detect_level); cmd->cca_detect_margin = __cpu_to_le32(detect_margin); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev set adaptive cca params enable:%d detection level:%d detection margin:%d\n", enable, detect_level, detect_margin); return skb; } static void ath10k_wmi_fw_vdev_stats_extd_fill(const struct ath10k_fw_stats_vdev_extd *vdev, char *buf, u32 *length) { u32 len = *length; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; u32 val; len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "vdev id", vdev->vdev_id); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "ppdu aggr count", vdev->ppdu_aggr_cnt); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "ppdu noack", vdev->ppdu_noack); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "mpdu queued", vdev->mpdu_queued); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "ppdu nonaggr count", vdev->ppdu_nonaggr_cnt); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "mpdu sw requeued", vdev->mpdu_sw_requeued); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "mpdu success retry", vdev->mpdu_suc_retry); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "mpdu success multitry", vdev->mpdu_suc_multitry); len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "mpdu fail retry", vdev->mpdu_fail_retry); val = vdev->tx_ftm_suc; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "tx ftm success", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->tx_ftm_suc_retry; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "tx ftm success retry", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->tx_ftm_fail; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "tx ftm fail", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->rx_ftmr_cnt; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "rx ftm request count", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->rx_ftmr_dup_cnt; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "rx ftm request dup count", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->rx_iftmr_cnt; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "rx initial ftm req count", MS(val, WMI_VDEV_STATS_FTM_COUNT)); val = vdev->rx_iftmr_dup_cnt; if (val & WMI_VDEV_STATS_FTM_COUNT_VALID) len += scnprintf(buf + len, buf_len - len, "%30s %u\n", "rx initial ftm req dup cnt", MS(val, WMI_VDEV_STATS_FTM_COUNT)); len += scnprintf(buf + len, buf_len - len, "\n"); *length = len; } void ath10k_wmi_10_4_op_fw_stats_fill(struct ath10k *ar, struct ath10k_fw_stats *fw_stats, char *buf) { u32 len = 0; u32 buf_len = ATH10K_FW_STATS_BUF_SIZE; const struct ath10k_fw_stats_pdev *pdev; const struct ath10k_fw_stats_vdev_extd *vdev; const struct ath10k_fw_stats_peer *peer; const struct ath10k_fw_extd_stats_peer *extd_peer; size_t num_peers; size_t num_vdevs; spin_lock_bh(&ar->data_lock); pdev = list_first_entry_or_null(&fw_stats->pdevs, struct ath10k_fw_stats_pdev, list); if (!pdev) { ath10k_warn(ar, "failed to get pdev stats\n"); goto unlock; } num_peers = list_count_nodes(&fw_stats->peers); num_vdevs = list_count_nodes(&fw_stats->vdevs); ath10k_wmi_fw_pdev_base_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_extra_stats_fill(pdev, buf, &len); ath10k_wmi_fw_pdev_tx_stats_fill(pdev, buf, &len); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "HW paused", pdev->hw_paused); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Seqs posted", pdev->seq_posted); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Seqs failed queueing", pdev->seq_failed_queueing); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Seqs completed", pdev->seq_completed); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Seqs restarted", pdev->seq_restarted); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MU Seqs posted", pdev->mu_seq_posted); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs SW flushed", pdev->mpdus_sw_flush); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs HW filtered", pdev->mpdus_hw_filter); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs truncated", pdev->mpdus_truncated); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs receive no ACK", pdev->mpdus_ack_failed); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "MPDUs expired", pdev->mpdus_expired); ath10k_wmi_fw_pdev_rx_stats_fill(pdev, buf, &len); len += scnprintf(buf + len, buf_len - len, "%30s %10d\n", "Num Rx Overflow errors", pdev->rx_ovfl_errs); len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k VDEV stats", num_vdevs); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(vdev, &fw_stats->vdevs, list) { ath10k_wmi_fw_vdev_stats_extd_fill(vdev, buf, &len); } len += scnprintf(buf + len, buf_len - len, "\n"); len += scnprintf(buf + len, buf_len - len, "%30s (%zu)\n", "ath10k PEER stats", num_peers); len += scnprintf(buf + len, buf_len - len, "%30s\n\n", "================="); list_for_each_entry(peer, &fw_stats->peers, list) { ath10k_wmi_fw_peer_stats_fill(peer, buf, &len, fw_stats->extended); } if (fw_stats->extended) { list_for_each_entry(extd_peer, &fw_stats->peers_extd, list) { ath10k_wmi_fw_extd_peer_stats_fill(extd_peer, buf, &len); } } unlock: spin_unlock_bh(&ar->data_lock); if (len >= buf_len) buf[len - 1] = 0; else buf[len] = 0; } int ath10k_wmi_op_get_vdev_subtype(struct ath10k *ar, enum wmi_vdev_subtype subtype) { switch (subtype) { case WMI_VDEV_SUBTYPE_NONE: return WMI_VDEV_SUBTYPE_LEGACY_NONE; case WMI_VDEV_SUBTYPE_P2P_DEVICE: return WMI_VDEV_SUBTYPE_LEGACY_P2P_DEV; case WMI_VDEV_SUBTYPE_P2P_CLIENT: return WMI_VDEV_SUBTYPE_LEGACY_P2P_CLI; case WMI_VDEV_SUBTYPE_P2P_GO: return WMI_VDEV_SUBTYPE_LEGACY_P2P_GO; case WMI_VDEV_SUBTYPE_PROXY_STA: return WMI_VDEV_SUBTYPE_LEGACY_PROXY_STA; case WMI_VDEV_SUBTYPE_MESH_11S: case WMI_VDEV_SUBTYPE_MESH_NON_11S: return -EOPNOTSUPP; } return -EOPNOTSUPP; } static int ath10k_wmi_10_2_4_op_get_vdev_subtype(struct ath10k *ar, enum wmi_vdev_subtype subtype) { switch (subtype) { case WMI_VDEV_SUBTYPE_NONE: return WMI_VDEV_SUBTYPE_10_2_4_NONE; case WMI_VDEV_SUBTYPE_P2P_DEVICE: return WMI_VDEV_SUBTYPE_10_2_4_P2P_DEV; case WMI_VDEV_SUBTYPE_P2P_CLIENT: return WMI_VDEV_SUBTYPE_10_2_4_P2P_CLI; case WMI_VDEV_SUBTYPE_P2P_GO: return WMI_VDEV_SUBTYPE_10_2_4_P2P_GO; case WMI_VDEV_SUBTYPE_PROXY_STA: return WMI_VDEV_SUBTYPE_10_2_4_PROXY_STA; case WMI_VDEV_SUBTYPE_MESH_11S: return WMI_VDEV_SUBTYPE_10_2_4_MESH_11S; case WMI_VDEV_SUBTYPE_MESH_NON_11S: return -EOPNOTSUPP; } return -EOPNOTSUPP; } static int ath10k_wmi_10_4_op_get_vdev_subtype(struct ath10k *ar, enum wmi_vdev_subtype subtype) { switch (subtype) { case WMI_VDEV_SUBTYPE_NONE: return WMI_VDEV_SUBTYPE_10_4_NONE; case WMI_VDEV_SUBTYPE_P2P_DEVICE: return WMI_VDEV_SUBTYPE_10_4_P2P_DEV; case WMI_VDEV_SUBTYPE_P2P_CLIENT: return WMI_VDEV_SUBTYPE_10_4_P2P_CLI; case WMI_VDEV_SUBTYPE_P2P_GO: return WMI_VDEV_SUBTYPE_10_4_P2P_GO; case WMI_VDEV_SUBTYPE_PROXY_STA: return WMI_VDEV_SUBTYPE_10_4_PROXY_STA; case WMI_VDEV_SUBTYPE_MESH_11S: return WMI_VDEV_SUBTYPE_10_4_MESH_11S; case WMI_VDEV_SUBTYPE_MESH_NON_11S: return WMI_VDEV_SUBTYPE_10_4_MESH_NON_11S; } return -EOPNOTSUPP; } static struct sk_buff * ath10k_wmi_10_4_ext_resource_config(struct ath10k *ar, enum wmi_host_platform_type type, u32 fw_feature_bitmap) { struct wmi_ext_resource_config_10_4_cmd *cmd; struct sk_buff *skb; u32 num_tdls_sleep_sta = 0; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); if (test_bit(WMI_SERVICE_TDLS_UAPSD_SLEEP_STA, ar->wmi.svc_map)) num_tdls_sleep_sta = TARGET_10_4_NUM_TDLS_SLEEP_STA; cmd = (struct wmi_ext_resource_config_10_4_cmd *)skb->data; cmd->host_platform_config = __cpu_to_le32(type); cmd->fw_feature_bitmap = __cpu_to_le32(fw_feature_bitmap); cmd->wlan_gpio_priority = __cpu_to_le32(ar->coex_gpio_pin); cmd->coex_version = __cpu_to_le32(WMI_NO_COEX_VERSION_SUPPORT); cmd->coex_gpio_pin1 = __cpu_to_le32(-1); cmd->coex_gpio_pin2 = __cpu_to_le32(-1); cmd->coex_gpio_pin3 = __cpu_to_le32(-1); cmd->num_tdls_vdevs = __cpu_to_le32(TARGET_10_4_NUM_TDLS_VDEVS); cmd->num_tdls_conn_table_entries = __cpu_to_le32(20); cmd->max_tdls_concurrent_sleep_sta = __cpu_to_le32(num_tdls_sleep_sta); cmd->max_tdls_concurrent_buffer_sta = __cpu_to_le32(TARGET_10_4_NUM_TDLS_BUFFER_STA); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi ext resource config host type %d firmware feature bitmap %08x\n", type, fw_feature_bitmap); return skb; } static struct sk_buff * ath10k_wmi_10_4_gen_update_fw_tdls_state(struct ath10k *ar, u32 vdev_id, enum wmi_tdls_state state) { struct wmi_10_4_tdls_set_state_cmd *cmd; struct sk_buff *skb; u32 options = 0; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); if (test_bit(WMI_SERVICE_TDLS_EXPLICIT_MODE_ONLY, ar->wmi.svc_map) && state == WMI_TDLS_ENABLE_ACTIVE) state = WMI_TDLS_ENABLE_PASSIVE; if (test_bit(WMI_SERVICE_TDLS_UAPSD_BUFFER_STA, ar->wmi.svc_map)) options |= WMI_TDLS_BUFFER_STA_EN; cmd = (struct wmi_10_4_tdls_set_state_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(vdev_id); cmd->state = __cpu_to_le32(state); cmd->notification_interval_ms = __cpu_to_le32(5000); cmd->tx_discovery_threshold = __cpu_to_le32(100); cmd->tx_teardown_threshold = __cpu_to_le32(5); cmd->rssi_teardown_threshold = __cpu_to_le32(-75); cmd->rssi_delta = __cpu_to_le32(-20); cmd->tdls_options = __cpu_to_le32(options); cmd->tdls_peer_traffic_ind_window = __cpu_to_le32(2); cmd->tdls_peer_traffic_response_timeout_ms = __cpu_to_le32(5000); cmd->tdls_puapsd_mask = __cpu_to_le32(0xf); cmd->tdls_puapsd_inactivity_time_ms = __cpu_to_le32(0); cmd->tdls_puapsd_rx_frame_threshold = __cpu_to_le32(10); cmd->teardown_notification_ms = __cpu_to_le32(10); cmd->tdls_peer_kickout_threshold = __cpu_to_le32(96); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi update fw tdls state %d for vdev %i\n", state, vdev_id); return skb; } static u32 ath10k_wmi_prepare_peer_qos(u8 uapsd_queues, u8 sp) { u32 peer_qos = 0; if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VO) peer_qos |= WMI_TDLS_PEER_QOS_AC_VO; if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_VI) peer_qos |= WMI_TDLS_PEER_QOS_AC_VI; if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BK) peer_qos |= WMI_TDLS_PEER_QOS_AC_BK; if (uapsd_queues & IEEE80211_WMM_IE_STA_QOSINFO_AC_BE) peer_qos |= WMI_TDLS_PEER_QOS_AC_BE; peer_qos |= SM(sp, WMI_TDLS_PEER_SP); return peer_qos; } static struct sk_buff * ath10k_wmi_10_4_op_gen_pdev_get_tpc_table_cmdid(struct ath10k *ar, u32 param) { struct wmi_pdev_get_tpc_table_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_get_tpc_table_cmd *)skb->data; cmd->param = __cpu_to_le32(param); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev get tpc table param:%d\n", param); return skb; } static struct sk_buff * ath10k_wmi_10_4_gen_tdls_peer_update(struct ath10k *ar, const struct wmi_tdls_peer_update_cmd_arg *arg, const struct wmi_tdls_peer_capab_arg *cap, const struct wmi_channel_arg *chan_arg) { struct wmi_10_4_tdls_peer_update_cmd *cmd; struct wmi_tdls_peer_capabilities *peer_cap; struct wmi_channel *chan; struct sk_buff *skb; u32 peer_qos; int len, chan_len; int i; /* tdls peer update cmd has place holder for one channel*/ chan_len = cap->peer_chan_len ? (cap->peer_chan_len - 1) : 0; len = sizeof(*cmd) + chan_len * sizeof(*chan); skb = ath10k_wmi_alloc_skb(ar, len); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_10_4_tdls_peer_update_cmd *)skb->data; cmd->vdev_id = __cpu_to_le32(arg->vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, arg->addr); cmd->peer_state = __cpu_to_le32(arg->peer_state); peer_qos = ath10k_wmi_prepare_peer_qos(cap->peer_uapsd_queues, cap->peer_max_sp); peer_cap = &cmd->peer_capab; peer_cap->peer_qos = __cpu_to_le32(peer_qos); peer_cap->buff_sta_support = __cpu_to_le32(cap->buff_sta_support); peer_cap->off_chan_support = __cpu_to_le32(cap->off_chan_support); peer_cap->peer_curr_operclass = __cpu_to_le32(cap->peer_curr_operclass); peer_cap->self_curr_operclass = __cpu_to_le32(cap->self_curr_operclass); peer_cap->peer_chan_len = __cpu_to_le32(cap->peer_chan_len); peer_cap->peer_operclass_len = __cpu_to_le32(cap->peer_operclass_len); for (i = 0; i < WMI_TDLS_MAX_SUPP_OPER_CLASSES; i++) peer_cap->peer_operclass[i] = cap->peer_operclass[i]; peer_cap->is_peer_responder = __cpu_to_le32(cap->is_peer_responder); peer_cap->pref_offchan_num = __cpu_to_le32(cap->pref_offchan_num); peer_cap->pref_offchan_bw = __cpu_to_le32(cap->pref_offchan_bw); for (i = 0; i < cap->peer_chan_len; i++) { chan = (struct wmi_channel *)&peer_cap->peer_chan_list[i]; ath10k_wmi_put_wmi_channel(ar, chan, &chan_arg[i]); } ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi tdls peer update vdev %i state %d n_chans %u\n", arg->vdev_id, arg->peer_state, cap->peer_chan_len); return skb; } static struct sk_buff * ath10k_wmi_10_4_gen_radar_found(struct ath10k *ar, const struct ath10k_radar_found_info *arg) { struct wmi_radar_found_info *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_radar_found_info *)skb->data; cmd->pri_min = __cpu_to_le32(arg->pri_min); cmd->pri_max = __cpu_to_le32(arg->pri_max); cmd->width_min = __cpu_to_le32(arg->width_min); cmd->width_max = __cpu_to_le32(arg->width_max); cmd->sidx_min = __cpu_to_le32(arg->sidx_min); cmd->sidx_max = __cpu_to_le32(arg->sidx_max); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi radar found pri_min %d pri_max %d width_min %d width_max %d sidx_min %d sidx_max %d\n", arg->pri_min, arg->pri_max, arg->width_min, arg->width_max, arg->sidx_min, arg->sidx_max); return skb; } static struct sk_buff * ath10k_wmi_10_4_gen_per_peer_per_tid_cfg(struct ath10k *ar, const struct wmi_per_peer_per_tid_cfg_arg *arg) { struct wmi_peer_per_tid_cfg_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); memset(skb->data, 0, sizeof(*cmd)); cmd = (struct wmi_peer_per_tid_cfg_cmd *)skb->data; cmd->vdev_id = cpu_to_le32(arg->vdev_id); ether_addr_copy(cmd->peer_macaddr.addr, arg->peer_macaddr.addr); cmd->tid = cpu_to_le32(arg->tid); cmd->ack_policy = cpu_to_le32(arg->ack_policy); cmd->aggr_control = cpu_to_le32(arg->aggr_control); cmd->rate_control = cpu_to_le32(arg->rate_ctrl); cmd->retry_count = cpu_to_le32(arg->retry_count); cmd->rcode_flags = cpu_to_le32(arg->rcode_flags); cmd->ext_tid_cfg_bitmap = cpu_to_le32(arg->ext_tid_cfg_bitmap); cmd->rtscts_ctrl = cpu_to_le32(arg->rtscts_ctrl); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi noack tid %d vdev id %d ack_policy %d aggr %u rate_ctrl %u rcflag %u retry_count %d rtscts %d ext_tid_cfg_bitmap %d mac_addr %pM\n", arg->tid, arg->vdev_id, arg->ack_policy, arg->aggr_control, arg->rate_ctrl, arg->rcode_flags, arg->retry_count, arg->rtscts_ctrl, arg->ext_tid_cfg_bitmap, arg->peer_macaddr.addr); return skb; } static struct sk_buff * ath10k_wmi_op_gen_echo(struct ath10k *ar, u32 value) { struct wmi_echo_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_echo_cmd *)skb->data; cmd->value = cpu_to_le32(value); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi echo value 0x%08x\n", value); return skb; } int ath10k_wmi_barrier(struct ath10k *ar) { int ret; int time_left; spin_lock_bh(&ar->data_lock); reinit_completion(&ar->wmi.barrier); spin_unlock_bh(&ar->data_lock); ret = ath10k_wmi_echo(ar, ATH10K_WMI_BARRIER_ECHO_ID); if (ret) { ath10k_warn(ar, "failed to submit wmi echo: %d\n", ret); return ret; } time_left = wait_for_completion_timeout(&ar->wmi.barrier, ATH10K_WMI_BARRIER_TIMEOUT_HZ); if (!time_left) return -ETIMEDOUT; return 0; } static struct sk_buff * ath10k_wmi_10_2_4_op_gen_bb_timing(struct ath10k *ar, const struct wmi_bb_timing_cfg_arg *arg) { struct wmi_pdev_bb_timing_cfg_cmd *cmd; struct sk_buff *skb; skb = ath10k_wmi_alloc_skb(ar, sizeof(*cmd)); if (!skb) return ERR_PTR(-ENOMEM); cmd = (struct wmi_pdev_bb_timing_cfg_cmd *)skb->data; cmd->bb_tx_timing = __cpu_to_le32(arg->bb_tx_timing); cmd->bb_xpa_timing = __cpu_to_le32(arg->bb_xpa_timing); ath10k_dbg(ar, ATH10K_DBG_WMI, "wmi pdev bb_tx_timing 0x%x bb_xpa_timing 0x%x\n", arg->bb_tx_timing, arg->bb_xpa_timing); return skb; } static const struct wmi_ops wmi_ops = { .rx = ath10k_wmi_op_rx, .map_svc = wmi_main_svc_map, .pull_scan = ath10k_wmi_op_pull_scan_ev, .pull_mgmt_rx = ath10k_wmi_op_pull_mgmt_rx_ev, .pull_ch_info = ath10k_wmi_op_pull_ch_info_ev, .pull_vdev_start = ath10k_wmi_op_pull_vdev_start_ev, .pull_peer_kick = ath10k_wmi_op_pull_peer_kick_ev, .pull_swba = ath10k_wmi_op_pull_swba_ev, .pull_phyerr_hdr = ath10k_wmi_op_pull_phyerr_ev_hdr, .pull_phyerr = ath10k_wmi_op_pull_phyerr_ev, .pull_svc_rdy = ath10k_wmi_main_op_pull_svc_rdy_ev, .pull_rdy = ath10k_wmi_op_pull_rdy_ev, .pull_fw_stats = ath10k_wmi_main_op_pull_fw_stats, .pull_roam_ev = ath10k_wmi_op_pull_roam_ev, .pull_echo_ev = ath10k_wmi_op_pull_echo_ev, .gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend, .gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume, .gen_pdev_set_rd = ath10k_wmi_op_gen_pdev_set_rd, .gen_pdev_set_param = ath10k_wmi_op_gen_pdev_set_param, .gen_init = ath10k_wmi_op_gen_init, .gen_start_scan = ath10k_wmi_op_gen_start_scan, .gen_stop_scan = ath10k_wmi_op_gen_stop_scan, .gen_vdev_create = ath10k_wmi_op_gen_vdev_create, .gen_vdev_delete = ath10k_wmi_op_gen_vdev_delete, .gen_vdev_start = ath10k_wmi_op_gen_vdev_start, .gen_vdev_stop = ath10k_wmi_op_gen_vdev_stop, .gen_vdev_up = ath10k_wmi_op_gen_vdev_up, .gen_vdev_down = ath10k_wmi_op_gen_vdev_down, .gen_vdev_set_param = ath10k_wmi_op_gen_vdev_set_param, .gen_vdev_install_key = ath10k_wmi_op_gen_vdev_install_key, .gen_vdev_spectral_conf = ath10k_wmi_op_gen_vdev_spectral_conf, .gen_vdev_spectral_enable = ath10k_wmi_op_gen_vdev_spectral_enable, /* .gen_vdev_wmm_conf not implemented */ .gen_peer_create = ath10k_wmi_op_gen_peer_create, .gen_peer_delete = ath10k_wmi_op_gen_peer_delete, .gen_peer_flush = ath10k_wmi_op_gen_peer_flush, .gen_peer_set_param = ath10k_wmi_op_gen_peer_set_param, .gen_peer_assoc = ath10k_wmi_op_gen_peer_assoc, .gen_set_psmode = ath10k_wmi_op_gen_set_psmode, .gen_set_sta_ps = ath10k_wmi_op_gen_set_sta_ps, .gen_set_ap_ps = ath10k_wmi_op_gen_set_ap_ps, .gen_scan_chan_list = ath10k_wmi_op_gen_scan_chan_list, .gen_beacon_dma = ath10k_wmi_op_gen_beacon_dma, .gen_pdev_set_wmm = ath10k_wmi_op_gen_pdev_set_wmm, .gen_request_stats = ath10k_wmi_op_gen_request_stats, .gen_force_fw_hang = ath10k_wmi_op_gen_force_fw_hang, .gen_mgmt_tx = ath10k_wmi_op_gen_mgmt_tx, .gen_dbglog_cfg = ath10k_wmi_op_gen_dbglog_cfg, .gen_pktlog_enable = ath10k_wmi_op_gen_pktlog_enable, .gen_pktlog_disable = ath10k_wmi_op_gen_pktlog_disable, .gen_pdev_set_quiet_mode = ath10k_wmi_op_gen_pdev_set_quiet_mode, /* .gen_pdev_get_temperature not implemented */ .gen_addba_clear_resp = ath10k_wmi_op_gen_addba_clear_resp, .gen_addba_send = ath10k_wmi_op_gen_addba_send, .gen_addba_set_resp = ath10k_wmi_op_gen_addba_set_resp, .gen_delba_send = ath10k_wmi_op_gen_delba_send, .fw_stats_fill = ath10k_wmi_main_op_fw_stats_fill, .get_vdev_subtype = ath10k_wmi_op_get_vdev_subtype, .gen_echo = ath10k_wmi_op_gen_echo, .gen_gpio_config = ath10k_wmi_op_gen_gpio_config, .gen_gpio_output = ath10k_wmi_op_gen_gpio_output, /* .gen_bcn_tmpl not implemented */ /* .gen_prb_tmpl not implemented */ /* .gen_p2p_go_bcn_ie not implemented */ /* .gen_adaptive_qcs not implemented */ /* .gen_pdev_enable_adaptive_cca not implemented */ }; static const struct wmi_ops wmi_10_1_ops = { .rx = ath10k_wmi_10_1_op_rx, .map_svc = wmi_10x_svc_map, .pull_svc_rdy = ath10k_wmi_10x_op_pull_svc_rdy_ev, .pull_fw_stats = ath10k_wmi_10x_op_pull_fw_stats, .gen_init = ath10k_wmi_10_1_op_gen_init, .gen_pdev_set_rd = ath10k_wmi_10x_op_gen_pdev_set_rd, .gen_start_scan = ath10k_wmi_10x_op_gen_start_scan, .gen_peer_assoc = ath10k_wmi_10_1_op_gen_peer_assoc, /* .gen_pdev_get_temperature not implemented */ /* shared with main branch */ .pull_scan = ath10k_wmi_op_pull_scan_ev, .pull_mgmt_rx = ath10k_wmi_op_pull_mgmt_rx_ev, .pull_ch_info = ath10k_wmi_op_pull_ch_info_ev, .pull_vdev_start = ath10k_wmi_op_pull_vdev_start_ev, .pull_peer_kick = ath10k_wmi_op_pull_peer_kick_ev, .pull_swba = ath10k_wmi_op_pull_swba_ev, .pull_phyerr_hdr = ath10k_wmi_op_pull_phyerr_ev_hdr, .pull_phyerr = ath10k_wmi_op_pull_phyerr_ev, .pull_rdy = ath10k_wmi_op_pull_rdy_ev, .pull_roam_ev = ath10k_wmi_op_pull_roam_ev, .pull_echo_ev = ath10k_wmi_op_pull_echo_ev, .gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend, .gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume, .gen_pdev_set_param = ath10k_wmi_op_gen_pdev_set_param, .gen_stop_scan = ath10k_wmi_op_gen_stop_scan, .gen_vdev_create = ath10k_wmi_op_gen_vdev_create, .gen_vdev_delete = ath10k_wmi_op_gen_vdev_delete, .gen_vdev_start = ath10k_wmi_op_gen_vdev_start, .gen_vdev_stop = ath10k_wmi_op_gen_vdev_stop, .gen_vdev_up = ath10k_wmi_op_gen_vdev_up, .gen_vdev_down = ath10k_wmi_op_gen_vdev_down, .gen_vdev_set_param = ath10k_wmi_op_gen_vdev_set_param, .gen_vdev_install_key = ath10k_wmi_op_gen_vdev_install_key, .gen_vdev_spectral_conf = ath10k_wmi_op_gen_vdev_spectral_conf, .gen_vdev_spectral_enable = ath10k_wmi_op_gen_vdev_spectral_enable, /* .gen_vdev_wmm_conf not implemented */ .gen_peer_create = ath10k_wmi_op_gen_peer_create, .gen_peer_delete = ath10k_wmi_op_gen_peer_delete, .gen_peer_flush = ath10k_wmi_op_gen_peer_flush, .gen_peer_set_param = ath10k_wmi_op_gen_peer_set_param, .gen_set_psmode = ath10k_wmi_op_gen_set_psmode, .gen_set_sta_ps = ath10k_wmi_op_gen_set_sta_ps, .gen_set_ap_ps = ath10k_wmi_op_gen_set_ap_ps, .gen_scan_chan_list = ath10k_wmi_op_gen_scan_chan_list, .gen_beacon_dma = ath10k_wmi_op_gen_beacon_dma, .gen_pdev_set_wmm = ath10k_wmi_op_gen_pdev_set_wmm, .gen_request_stats = ath10k_wmi_op_gen_request_stats, .gen_force_fw_hang = ath10k_wmi_op_gen_force_fw_hang, .gen_mgmt_tx = ath10k_wmi_op_gen_mgmt_tx, .gen_dbglog_cfg = ath10k_wmi_op_gen_dbglog_cfg, .gen_pktlog_enable = ath10k_wmi_op_gen_pktlog_enable, .gen_pktlog_disable = ath10k_wmi_op_gen_pktlog_disable, .gen_pdev_set_quiet_mode = ath10k_wmi_op_gen_pdev_set_quiet_mode, .gen_addba_clear_resp = ath10k_wmi_op_gen_addba_clear_resp, .gen_addba_send = ath10k_wmi_op_gen_addba_send, .gen_addba_set_resp = ath10k_wmi_op_gen_addba_set_resp, .gen_delba_send = ath10k_wmi_op_gen_delba_send, .fw_stats_fill = ath10k_wmi_10x_op_fw_stats_fill, .get_vdev_subtype = ath10k_wmi_op_get_vdev_subtype, .gen_echo = ath10k_wmi_op_gen_echo, .gen_gpio_config = ath10k_wmi_op_gen_gpio_config, .gen_gpio_output = ath10k_wmi_op_gen_gpio_output, /* .gen_bcn_tmpl not implemented */ /* .gen_prb_tmpl not implemented */ /* .gen_p2p_go_bcn_ie not implemented */ /* .gen_adaptive_qcs not implemented */ /* .gen_pdev_enable_adaptive_cca not implemented */ }; static const struct wmi_ops wmi_10_2_ops = { .rx = ath10k_wmi_10_2_op_rx, .pull_fw_stats = ath10k_wmi_10_2_op_pull_fw_stats, .gen_init = ath10k_wmi_10_2_op_gen_init, .gen_peer_assoc = ath10k_wmi_10_2_op_gen_peer_assoc, /* .gen_pdev_get_temperature not implemented */ /* shared with 10.1 */ .map_svc = wmi_10x_svc_map, .pull_svc_rdy = ath10k_wmi_10x_op_pull_svc_rdy_ev, .gen_pdev_set_rd = ath10k_wmi_10x_op_gen_pdev_set_rd, .gen_start_scan = ath10k_wmi_10x_op_gen_start_scan, .gen_echo = ath10k_wmi_op_gen_echo, .pull_scan = ath10k_wmi_op_pull_scan_ev, .pull_mgmt_rx = ath10k_wmi_op_pull_mgmt_rx_ev, .pull_ch_info = ath10k_wmi_op_pull_ch_info_ev, .pull_vdev_start = ath10k_wmi_op_pull_vdev_start_ev, .pull_peer_kick = ath10k_wmi_op_pull_peer_kick_ev, .pull_swba = ath10k_wmi_op_pull_swba_ev, .pull_phyerr_hdr = ath10k_wmi_op_pull_phyerr_ev_hdr, .pull_phyerr = ath10k_wmi_op_pull_phyerr_ev, .pull_rdy = ath10k_wmi_op_pull_rdy_ev, .pull_roam_ev = ath10k_wmi_op_pull_roam_ev, .pull_echo_ev = ath10k_wmi_op_pull_echo_ev, .gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend, .gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume, .gen_pdev_set_param = ath10k_wmi_op_gen_pdev_set_param, .gen_stop_scan = ath10k_wmi_op_gen_stop_scan, .gen_vdev_create = ath10k_wmi_op_gen_vdev_create, .gen_vdev_delete = ath10k_wmi_op_gen_vdev_delete, .gen_vdev_start = ath10k_wmi_op_gen_vdev_start, .gen_vdev_stop = ath10k_wmi_op_gen_vdev_stop, .gen_vdev_up = ath10k_wmi_op_gen_vdev_up, .gen_vdev_down = ath10k_wmi_op_gen_vdev_down, .gen_vdev_set_param = ath10k_wmi_op_gen_vdev_set_param, .gen_vdev_install_key = ath10k_wmi_op_gen_vdev_install_key, .gen_vdev_spectral_conf = ath10k_wmi_op_gen_vdev_spectral_conf, .gen_vdev_spectral_enable = ath10k_wmi_op_gen_vdev_spectral_enable, /* .gen_vdev_wmm_conf not implemented */ .gen_peer_create = ath10k_wmi_op_gen_peer_create, .gen_peer_delete = ath10k_wmi_op_gen_peer_delete, .gen_peer_flush = ath10k_wmi_op_gen_peer_flush, .gen_pdev_set_base_macaddr = ath10k_wmi_op_gen_pdev_set_base_macaddr, .gen_peer_set_param = ath10k_wmi_op_gen_peer_set_param, .gen_set_psmode = ath10k_wmi_op_gen_set_psmode, .gen_set_sta_ps = ath10k_wmi_op_gen_set_sta_ps, .gen_set_ap_ps = ath10k_wmi_op_gen_set_ap_ps, .gen_scan_chan_list = ath10k_wmi_op_gen_scan_chan_list, .gen_beacon_dma = ath10k_wmi_op_gen_beacon_dma, .gen_pdev_set_wmm = ath10k_wmi_op_gen_pdev_set_wmm, .gen_request_stats = ath10k_wmi_op_gen_request_stats, .gen_force_fw_hang = ath10k_wmi_op_gen_force_fw_hang, .gen_mgmt_tx = ath10k_wmi_op_gen_mgmt_tx, .gen_dbglog_cfg = ath10k_wmi_op_gen_dbglog_cfg, .gen_pktlog_enable = ath10k_wmi_op_gen_pktlog_enable, .gen_pktlog_disable = ath10k_wmi_op_gen_pktlog_disable, .gen_pdev_set_quiet_mode = ath10k_wmi_op_gen_pdev_set_quiet_mode, .gen_addba_clear_resp = ath10k_wmi_op_gen_addba_clear_resp, .gen_addba_send = ath10k_wmi_op_gen_addba_send, .gen_addba_set_resp = ath10k_wmi_op_gen_addba_set_resp, .gen_delba_send = ath10k_wmi_op_gen_delba_send, .fw_stats_fill = ath10k_wmi_10x_op_fw_stats_fill, .get_vdev_subtype = ath10k_wmi_op_get_vdev_subtype, .gen_gpio_config = ath10k_wmi_op_gen_gpio_config, .gen_gpio_output = ath10k_wmi_op_gen_gpio_output, /* .gen_pdev_enable_adaptive_cca not implemented */ }; static const struct wmi_ops wmi_10_2_4_ops = { .rx = ath10k_wmi_10_2_op_rx, .pull_fw_stats = ath10k_wmi_10_2_4_op_pull_fw_stats, .gen_init = ath10k_wmi_10_2_op_gen_init, .gen_peer_assoc = ath10k_wmi_10_2_op_gen_peer_assoc, .gen_pdev_get_temperature = ath10k_wmi_10_2_op_gen_pdev_get_temperature, .gen_pdev_bss_chan_info_req = ath10k_wmi_10_2_op_gen_pdev_bss_chan_info, /* shared with 10.1 */ .map_svc = wmi_10x_svc_map, .pull_svc_rdy = ath10k_wmi_10x_op_pull_svc_rdy_ev, .gen_pdev_set_rd = ath10k_wmi_10x_op_gen_pdev_set_rd, .gen_start_scan = ath10k_wmi_10x_op_gen_start_scan, .gen_echo = ath10k_wmi_op_gen_echo, .pull_scan = ath10k_wmi_op_pull_scan_ev, .pull_mgmt_rx = ath10k_wmi_op_pull_mgmt_rx_ev, .pull_ch_info = ath10k_wmi_op_pull_ch_info_ev, .pull_vdev_start = ath10k_wmi_op_pull_vdev_start_ev, .pull_peer_kick = ath10k_wmi_op_pull_peer_kick_ev, .pull_swba = ath10k_wmi_10_2_4_op_pull_swba_ev, .pull_phyerr_hdr = ath10k_wmi_op_pull_phyerr_ev_hdr, .pull_phyerr = ath10k_wmi_op_pull_phyerr_ev, .pull_rdy = ath10k_wmi_op_pull_rdy_ev, .pull_roam_ev = ath10k_wmi_op_pull_roam_ev, .pull_echo_ev = ath10k_wmi_op_pull_echo_ev, .gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend, .gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume, .gen_pdev_set_param = ath10k_wmi_op_gen_pdev_set_param, .gen_stop_scan = ath10k_wmi_op_gen_stop_scan, .gen_vdev_create = ath10k_wmi_op_gen_vdev_create, .gen_vdev_delete = ath10k_wmi_op_gen_vdev_delete, .gen_vdev_start = ath10k_wmi_op_gen_vdev_start, .gen_vdev_stop = ath10k_wmi_op_gen_vdev_stop, .gen_vdev_up = ath10k_wmi_op_gen_vdev_up, .gen_vdev_down = ath10k_wmi_op_gen_vdev_down, .gen_vdev_set_param = ath10k_wmi_op_gen_vdev_set_param, .gen_vdev_install_key = ath10k_wmi_op_gen_vdev_install_key, .gen_vdev_spectral_conf = ath10k_wmi_op_gen_vdev_spectral_conf, .gen_vdev_spectral_enable = ath10k_wmi_op_gen_vdev_spectral_enable, .gen_peer_create = ath10k_wmi_op_gen_peer_create, .gen_peer_delete = ath10k_wmi_op_gen_peer_delete, .gen_peer_flush = ath10k_wmi_op_gen_peer_flush, .gen_peer_set_param = ath10k_wmi_op_gen_peer_set_param, .gen_set_psmode = ath10k_wmi_op_gen_set_psmode, .gen_set_sta_ps = ath10k_wmi_op_gen_set_sta_ps, .gen_set_ap_ps = ath10k_wmi_op_gen_set_ap_ps, .gen_scan_chan_list = ath10k_wmi_op_gen_scan_chan_list, .gen_beacon_dma = ath10k_wmi_op_gen_beacon_dma, .gen_pdev_set_wmm = ath10k_wmi_op_gen_pdev_set_wmm, .gen_request_stats = ath10k_wmi_op_gen_request_stats, .gen_force_fw_hang = ath10k_wmi_op_gen_force_fw_hang, .gen_mgmt_tx = ath10k_wmi_op_gen_mgmt_tx, .gen_dbglog_cfg = ath10k_wmi_op_gen_dbglog_cfg, .gen_pktlog_enable = ath10k_wmi_op_gen_pktlog_enable, .gen_pktlog_disable = ath10k_wmi_op_gen_pktlog_disable, .gen_pdev_set_quiet_mode = ath10k_wmi_op_gen_pdev_set_quiet_mode, .gen_addba_clear_resp = ath10k_wmi_op_gen_addba_clear_resp, .gen_addba_send = ath10k_wmi_op_gen_addba_send, .gen_addba_set_resp = ath10k_wmi_op_gen_addba_set_resp, .gen_delba_send = ath10k_wmi_op_gen_delba_send, .gen_pdev_get_tpc_config = ath10k_wmi_10_2_4_op_gen_pdev_get_tpc_config, .fw_stats_fill = ath10k_wmi_10x_op_fw_stats_fill, .gen_pdev_enable_adaptive_cca = ath10k_wmi_op_gen_pdev_enable_adaptive_cca, .get_vdev_subtype = ath10k_wmi_10_2_4_op_get_vdev_subtype, .gen_bb_timing = ath10k_wmi_10_2_4_op_gen_bb_timing, .gen_gpio_config = ath10k_wmi_op_gen_gpio_config, .gen_gpio_output = ath10k_wmi_op_gen_gpio_output, /* .gen_bcn_tmpl not implemented */ /* .gen_prb_tmpl not implemented */ /* .gen_p2p_go_bcn_ie not implemented */ /* .gen_adaptive_qcs not implemented */ }; static const struct wmi_ops wmi_10_4_ops = { .rx = ath10k_wmi_10_4_op_rx, .map_svc = wmi_10_4_svc_map, .pull_fw_stats = ath10k_wmi_10_4_op_pull_fw_stats, .pull_scan = ath10k_wmi_op_pull_scan_ev, .pull_mgmt_rx = ath10k_wmi_10_4_op_pull_mgmt_rx_ev, .pull_ch_info = ath10k_wmi_10_4_op_pull_ch_info_ev, .pull_vdev_start = ath10k_wmi_op_pull_vdev_start_ev, .pull_peer_kick = ath10k_wmi_op_pull_peer_kick_ev, .pull_swba = ath10k_wmi_10_4_op_pull_swba_ev, .pull_phyerr_hdr = ath10k_wmi_10_4_op_pull_phyerr_ev_hdr, .pull_phyerr = ath10k_wmi_10_4_op_pull_phyerr_ev, .pull_svc_rdy = ath10k_wmi_main_op_pull_svc_rdy_ev, .pull_rdy = ath10k_wmi_op_pull_rdy_ev, .pull_roam_ev = ath10k_wmi_op_pull_roam_ev, .pull_dfs_status_ev = ath10k_wmi_10_4_op_pull_dfs_status_ev, .get_txbf_conf_scheme = ath10k_wmi_10_4_txbf_conf_scheme, .gen_pdev_suspend = ath10k_wmi_op_gen_pdev_suspend, .gen_pdev_resume = ath10k_wmi_op_gen_pdev_resume, .gen_pdev_set_base_macaddr = ath10k_wmi_op_gen_pdev_set_base_macaddr, .gen_pdev_set_rd = ath10k_wmi_10x_op_gen_pdev_set_rd, .gen_pdev_set_param = ath10k_wmi_op_gen_pdev_set_param, .gen_init = ath10k_wmi_10_4_op_gen_init, .gen_start_scan = ath10k_wmi_op_gen_start_scan, .gen_stop_scan = ath10k_wmi_op_gen_stop_scan, .gen_vdev_create = ath10k_wmi_op_gen_vdev_create, .gen_vdev_delete = ath10k_wmi_op_gen_vdev_delete, .gen_vdev_start = ath10k_wmi_op_gen_vdev_start, .gen_vdev_stop = ath10k_wmi_op_gen_vdev_stop, .gen_vdev_up = ath10k_wmi_op_gen_vdev_up, .gen_vdev_down = ath10k_wmi_op_gen_vdev_down, .gen_vdev_set_param = ath10k_wmi_op_gen_vdev_set_param, .gen_vdev_install_key = ath10k_wmi_op_gen_vdev_install_key, .gen_vdev_spectral_conf = ath10k_wmi_op_gen_vdev_spectral_conf, .gen_vdev_spectral_enable = ath10k_wmi_op_gen_vdev_spectral_enable, .gen_peer_create = ath10k_wmi_op_gen_peer_create, .gen_peer_delete = ath10k_wmi_op_gen_peer_delete, .gen_peer_flush = ath10k_wmi_op_gen_peer_flush, .gen_peer_set_param = ath10k_wmi_op_gen_peer_set_param, .gen_peer_assoc = ath10k_wmi_10_4_op_gen_peer_assoc, .gen_set_psmode = ath10k_wmi_op_gen_set_psmode, .gen_set_sta_ps = ath10k_wmi_op_gen_set_sta_ps, .gen_set_ap_ps = ath10k_wmi_op_gen_set_ap_ps, .gen_scan_chan_list = ath10k_wmi_op_gen_scan_chan_list, .gen_beacon_dma = ath10k_wmi_op_gen_beacon_dma, .gen_pdev_set_wmm = ath10k_wmi_op_gen_pdev_set_wmm, .gen_force_fw_hang = ath10k_wmi_op_gen_force_fw_hang, .gen_mgmt_tx = ath10k_wmi_op_gen_mgmt_tx, .gen_dbglog_cfg = ath10k_wmi_10_4_op_gen_dbglog_cfg, .gen_pktlog_enable = ath10k_wmi_op_gen_pktlog_enable, .gen_pktlog_disable = ath10k_wmi_op_gen_pktlog_disable, .gen_pdev_set_quiet_mode = ath10k_wmi_op_gen_pdev_set_quiet_mode, .gen_addba_clear_resp = ath10k_wmi_op_gen_addba_clear_resp, .gen_addba_send = ath10k_wmi_op_gen_addba_send, .gen_addba_set_resp = ath10k_wmi_op_gen_addba_set_resp, .gen_delba_send = ath10k_wmi_op_gen_delba_send, .fw_stats_fill = ath10k_wmi_10_4_op_fw_stats_fill, .ext_resource_config = ath10k_wmi_10_4_ext_resource_config, .gen_update_fw_tdls_state = ath10k_wmi_10_4_gen_update_fw_tdls_state, .gen_tdls_peer_update = ath10k_wmi_10_4_gen_tdls_peer_update, .gen_pdev_get_tpc_table_cmdid = ath10k_wmi_10_4_op_gen_pdev_get_tpc_table_cmdid, .gen_radar_found = ath10k_wmi_10_4_gen_radar_found, .gen_per_peer_per_tid_cfg = ath10k_wmi_10_4_gen_per_peer_per_tid_cfg, /* shared with 10.2 */ .pull_echo_ev = ath10k_wmi_op_pull_echo_ev, .gen_request_stats = ath10k_wmi_op_gen_request_stats, .gen_pdev_get_temperature = ath10k_wmi_10_2_op_gen_pdev_get_temperature, .get_vdev_subtype = ath10k_wmi_10_4_op_get_vdev_subtype, .gen_pdev_bss_chan_info_req = ath10k_wmi_10_2_op_gen_pdev_bss_chan_info, .gen_echo = ath10k_wmi_op_gen_echo, .gen_pdev_get_tpc_config = ath10k_wmi_10_2_4_op_gen_pdev_get_tpc_config, .gen_gpio_config = ath10k_wmi_op_gen_gpio_config, .gen_gpio_output = ath10k_wmi_op_gen_gpio_output, }; int ath10k_wmi_attach(struct ath10k *ar) { switch (ar->running_fw->fw_file.wmi_op_version) { case ATH10K_FW_WMI_OP_VERSION_10_4: ar->wmi.ops = &wmi_10_4_ops; ar->wmi.cmd = &wmi_10_4_cmd_map; ar->wmi.vdev_param = &wmi_10_4_vdev_param_map; ar->wmi.pdev_param = &wmi_10_4_pdev_param_map; ar->wmi.peer_param = &wmi_peer_param_map; ar->wmi.peer_flags = &wmi_10_2_peer_flags_map; ar->wmi_key_cipher = wmi_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_10_2_4: ar->wmi.cmd = &wmi_10_2_4_cmd_map; ar->wmi.ops = &wmi_10_2_4_ops; ar->wmi.vdev_param = &wmi_10_2_4_vdev_param_map; ar->wmi.pdev_param = &wmi_10_2_4_pdev_param_map; ar->wmi.peer_param = &wmi_peer_param_map; ar->wmi.peer_flags = &wmi_10_2_peer_flags_map; ar->wmi_key_cipher = wmi_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_10_2: ar->wmi.cmd = &wmi_10_2_cmd_map; ar->wmi.ops = &wmi_10_2_ops; ar->wmi.vdev_param = &wmi_10x_vdev_param_map; ar->wmi.pdev_param = &wmi_10x_pdev_param_map; ar->wmi.peer_param = &wmi_peer_param_map; ar->wmi.peer_flags = &wmi_10_2_peer_flags_map; ar->wmi_key_cipher = wmi_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_10_1: ar->wmi.cmd = &wmi_10x_cmd_map; ar->wmi.ops = &wmi_10_1_ops; ar->wmi.vdev_param = &wmi_10x_vdev_param_map; ar->wmi.pdev_param = &wmi_10x_pdev_param_map; ar->wmi.peer_param = &wmi_peer_param_map; ar->wmi.peer_flags = &wmi_10x_peer_flags_map; ar->wmi_key_cipher = wmi_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_MAIN: ar->wmi.cmd = &wmi_cmd_map; ar->wmi.ops = &wmi_ops; ar->wmi.vdev_param = &wmi_vdev_param_map; ar->wmi.pdev_param = &wmi_pdev_param_map; ar->wmi.peer_param = &wmi_peer_param_map; ar->wmi.peer_flags = &wmi_peer_flags_map; ar->wmi_key_cipher = wmi_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_TLV: ath10k_wmi_tlv_attach(ar); ar->wmi_key_cipher = wmi_tlv_key_cipher_suites; break; case ATH10K_FW_WMI_OP_VERSION_UNSET: case ATH10K_FW_WMI_OP_VERSION_MAX: ath10k_err(ar, "unsupported WMI op version: %d\n", ar->running_fw->fw_file.wmi_op_version); return -EINVAL; } init_completion(&ar->wmi.service_ready); init_completion(&ar->wmi.unified_ready); init_completion(&ar->wmi.barrier); init_completion(&ar->wmi.radar_confirm); INIT_WORK(&ar->svc_rdy_work, ath10k_wmi_event_service_ready_work); INIT_WORK(&ar->radar_confirmation_work, ath10k_radar_confirmation_work); if (test_bit(ATH10K_FW_FEATURE_MGMT_TX_BY_REF, ar->running_fw->fw_file.fw_features)) { idr_init(&ar->wmi.mgmt_pending_tx); } return 0; } void ath10k_wmi_free_host_mem(struct ath10k *ar) { int i; /* free the host memory chunks requested by firmware */ for (i = 0; i < ar->wmi.num_mem_chunks; i++) { dma_free_coherent(ar->dev, ar->wmi.mem_chunks[i].len, ar->wmi.mem_chunks[i].vaddr, ar->wmi.mem_chunks[i].paddr); } ar->wmi.num_mem_chunks = 0; } static int ath10k_wmi_mgmt_tx_clean_up_pending(int msdu_id, void *ptr, void *ctx) { struct ath10k_mgmt_tx_pkt_addr *pkt_addr = ptr; struct ath10k *ar = ctx; struct sk_buff *msdu; ath10k_dbg(ar, ATH10K_DBG_WMI, "force cleanup mgmt msdu_id %u\n", msdu_id); msdu = pkt_addr->vaddr; dma_unmap_single(ar->dev, pkt_addr->paddr, msdu->len, DMA_TO_DEVICE); ieee80211_free_txskb(ar->hw, msdu); kfree(pkt_addr); return 0; } void ath10k_wmi_detach(struct ath10k *ar) { if (test_bit(ATH10K_FW_FEATURE_MGMT_TX_BY_REF, ar->running_fw->fw_file.fw_features)) { spin_lock_bh(&ar->data_lock); idr_for_each(&ar->wmi.mgmt_pending_tx, ath10k_wmi_mgmt_tx_clean_up_pending, ar); idr_destroy(&ar->wmi.mgmt_pending_tx); spin_unlock_bh(&ar->data_lock); } cancel_work_sync(&ar->svc_rdy_work); dev_kfree_skb(ar->svc_rdy_skb); } |
| 696 696 634 157 158 157 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 | // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2008 IBM Corporation * Author: Mimi Zohar <zohar@us.ibm.com> * * File: integrity_audit.c * Audit calls for the integrity subsystem */ #include <linux/fs.h> #include <linux/gfp.h> #include <linux/audit.h> #include "integrity.h" static int integrity_audit_info; /* ima_audit_setup - enable informational auditing messages */ static int __init integrity_audit_setup(char *str) { unsigned long audit; if (!kstrtoul(str, 0, &audit)) integrity_audit_info = audit ? 1 : 0; return 1; } __setup("integrity_audit=", integrity_audit_setup); void integrity_audit_msg(int audit_msgno, struct inode *inode, const unsigned char *fname, const char *op, const char *cause, int result, int audit_info) { integrity_audit_message(audit_msgno, inode, fname, op, cause, result, audit_info, 0); } void integrity_audit_message(int audit_msgno, struct inode *inode, const unsigned char *fname, const char *op, const char *cause, int result, int audit_info, int errno) { struct audit_buffer *ab; char name[TASK_COMM_LEN]; if (!integrity_audit_info && audit_info == 1) /* Skip info messages */ return; ab = audit_log_start(audit_context(), GFP_KERNEL, audit_msgno); if (!ab) return; audit_log_format(ab, "pid=%d uid=%u auid=%u ses=%u", task_pid_nr(current), from_kuid(&init_user_ns, current_uid()), from_kuid(&init_user_ns, audit_get_loginuid(current)), audit_get_sessionid(current)); audit_log_task_context(ab); audit_log_format(ab, " op=%s cause=%s comm=", op, cause); audit_log_untrustedstring(ab, get_task_comm(name, current)); if (fname) { audit_log_format(ab, " name="); audit_log_untrustedstring(ab, fname); } if (inode) { audit_log_format(ab, " dev="); audit_log_untrustedstring(ab, inode->i_sb->s_id); audit_log_format(ab, " ino=%lu", inode->i_ino); } audit_log_format(ab, " res=%d errno=%d", !result, errno); audit_log_end(ab); } |
| 1 1 1 2 2 2 2 6 5 1 3 2 3 3 1 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 | // SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2007, 2008 Karsten Wiese <fzu@wemgehoertderstaat.de> */ #include <linux/slab.h> #include <linux/usb.h> #include <linux/usb/audio.h> #include <linux/module.h> #include <sound/core.h> #include <sound/hwdep.h> #include <sound/pcm.h> #include <sound/initval.h> #define MODNAME "US122L" #include "usb_stream.c" #include "../usbaudio.h" #include "../midi.h" #include "us122l.h" MODULE_AUTHOR("Karsten Wiese <fzu@wemgehoertderstaat.de>"); MODULE_DESCRIPTION("TASCAM "NAME_ALLCAPS" Version 0.5"); MODULE_LICENSE("GPL"); static int index[SNDRV_CARDS] = SNDRV_DEFAULT_IDX; /* Index 0-max */ static char *id[SNDRV_CARDS] = SNDRV_DEFAULT_STR; /* Id for this card */ /* Enable this card */ static bool enable[SNDRV_CARDS] = SNDRV_DEFAULT_ENABLE_PNP; module_param_array(index, int, NULL, 0444); MODULE_PARM_DESC(index, "Index value for "NAME_ALLCAPS"."); module_param_array(id, charp, NULL, 0444); MODULE_PARM_DESC(id, "ID string for "NAME_ALLCAPS"."); module_param_array(enable, bool, NULL, 0444); MODULE_PARM_DESC(enable, "Enable "NAME_ALLCAPS"."); /* driver_info flags */ #define US122L_FLAG_US144 BIT(0) static int snd_us122l_card_used[SNDRV_CARDS]; static int us122l_create_usbmidi(struct snd_card *card) { static const struct snd_usb_midi_endpoint_info quirk_data = { .out_ep = 4, .in_ep = 3, .out_cables = 0x001, .in_cables = 0x001 }; static const struct snd_usb_audio_quirk quirk = { .vendor_name = "US122L", .product_name = NAME_ALLCAPS, .ifnum = 1, .type = QUIRK_MIDI_US122L, .data = &quirk_data }; struct usb_device *dev = US122L(card)->dev; struct usb_interface *iface = usb_ifnum_to_if(dev, 1); return snd_usbmidi_create(card, iface, &US122L(card)->midi_list, &quirk); } static int us144_create_usbmidi(struct snd_card *card) { static const struct snd_usb_midi_endpoint_info quirk_data = { .out_ep = 4, .in_ep = 3, .out_cables = 0x001, .in_cables = 0x001 }; static const struct snd_usb_audio_quirk quirk = { .vendor_name = "US144", .product_name = NAME_ALLCAPS, .ifnum = 0, .type = QUIRK_MIDI_US122L, .data = &quirk_data }; struct usb_device *dev = US122L(card)->dev; struct usb_interface *iface = usb_ifnum_to_if(dev, 0); return snd_usbmidi_create(card, iface, &US122L(card)->midi_list, &quirk); } static void pt_info_set(struct usb_device *dev, u8 v) { usb_control_msg_send(dev, 0, 'I', USB_DIR_OUT | USB_TYPE_VENDOR | USB_RECIP_DEVICE, v, 0, NULL, 0, 1000, GFP_NOIO); } static vm_fault_t usb_stream_hwdep_vm_fault(struct vm_fault *vmf) { unsigned long offset; struct page *page; void *vaddr; struct us122l *us122l = vmf->vma->vm_private_data; struct usb_stream *s; mutex_lock(&us122l->mutex); s = us122l->sk.s; if (!s) goto unlock; offset = vmf->pgoff << PAGE_SHIFT; if (offset < PAGE_ALIGN(s->read_size)) { vaddr = (char *)s + offset; } else { offset -= PAGE_ALIGN(s->read_size); if (offset >= PAGE_ALIGN(s->write_size)) goto unlock; vaddr = us122l->sk.write_page + offset; } page = virt_to_page(vaddr); get_page(page); mutex_unlock(&us122l->mutex); vmf->page = page; return 0; unlock: mutex_unlock(&us122l->mutex); return VM_FAULT_SIGBUS; } static const struct vm_operations_struct usb_stream_hwdep_vm_ops = { .fault = usb_stream_hwdep_vm_fault, }; static int usb_stream_hwdep_open(struct snd_hwdep *hw, struct file *file) { struct us122l *us122l = hw->private_data; struct usb_interface *iface; if (hw->used >= 2) return -EBUSY; if (!us122l->first) us122l->first = file; if (us122l->is_us144) { iface = usb_ifnum_to_if(us122l->dev, 0); usb_autopm_get_interface(iface); } iface = usb_ifnum_to_if(us122l->dev, 1); usb_autopm_get_interface(iface); return 0; } static int usb_stream_hwdep_release(struct snd_hwdep *hw, struct file *file) { struct us122l *us122l = hw->private_data; struct usb_interface *iface; if (us122l->is_us144) { iface = usb_ifnum_to_if(us122l->dev, 0); usb_autopm_put_interface(iface); } iface = usb_ifnum_to_if(us122l->dev, 1); usb_autopm_put_interface(iface); if (us122l->first == file) us122l->first = NULL; mutex_lock(&us122l->mutex); if (us122l->master == file) us122l->master = us122l->slave; us122l->slave = NULL; mutex_unlock(&us122l->mutex); return 0; } static int usb_stream_hwdep_mmap(struct snd_hwdep *hw, struct file *filp, struct vm_area_struct *area) { unsigned long size = area->vm_end - area->vm_start; struct us122l *us122l = hw->private_data; unsigned long offset; struct usb_stream *s; int err = 0; bool read; offset = area->vm_pgoff << PAGE_SHIFT; mutex_lock(&us122l->mutex); s = us122l->sk.s; read = offset < s->read_size; if (read && area->vm_flags & VM_WRITE) { err = -EPERM; goto out; } /* if userspace tries to mmap beyond end of our buffer, fail */ if (size > PAGE_ALIGN(read ? s->read_size : s->write_size)) { dev_warn(hw->card->dev, "%s: size %lu > %u\n", __func__, size, read ? s->read_size : s->write_size); err = -EINVAL; goto out; } area->vm_ops = &usb_stream_hwdep_vm_ops; vm_flags_set(area, VM_DONTDUMP); if (!read) vm_flags_set(area, VM_DONTEXPAND); area->vm_private_data = us122l; out: mutex_unlock(&us122l->mutex); return err; } static __poll_t usb_stream_hwdep_poll(struct snd_hwdep *hw, struct file *file, poll_table *wait) { struct us122l *us122l = hw->private_data; unsigned int *polled; __poll_t mask; poll_wait(file, &us122l->sk.sleep, wait); mask = EPOLLIN | EPOLLOUT | EPOLLWRNORM | EPOLLERR; if (mutex_trylock(&us122l->mutex)) { struct usb_stream *s = us122l->sk.s; if (s && s->state == usb_stream_ready) { if (us122l->first == file) polled = &s->periods_polled; else polled = &us122l->second_periods_polled; if (*polled != s->periods_done) { *polled = s->periods_done; mask = EPOLLIN | EPOLLOUT | EPOLLWRNORM; } else { mask = 0; } } mutex_unlock(&us122l->mutex); } return mask; } static void us122l_stop(struct us122l *us122l) { struct list_head *p; list_for_each(p, &us122l->midi_list) snd_usbmidi_input_stop(p); usb_stream_stop(&us122l->sk); usb_stream_free(&us122l->sk); } static int us122l_set_sample_rate(struct usb_device *dev, int rate) { unsigned int ep = 0x81; unsigned char data[3]; int err; data[0] = rate; data[1] = rate >> 8; data[2] = rate >> 16; err = usb_control_msg_send(dev, 0, UAC_SET_CUR, USB_TYPE_CLASS | USB_RECIP_ENDPOINT | USB_DIR_OUT, UAC_EP_CS_ATTR_SAMPLE_RATE << 8, ep, data, 3, 1000, GFP_NOIO); if (err) dev_err(&dev->dev, "%d: cannot set freq %d to ep 0x%x\n", dev->devnum, rate, ep); return err; } static bool us122l_start(struct us122l *us122l, unsigned int rate, unsigned int period_frames) { struct list_head *p; int err; unsigned int use_packsize = 0; bool success = false; if (us122l->dev->speed == USB_SPEED_HIGH) { /* The us-122l's descriptor defaults to iso max_packsize 78, which isn't needed for samplerates <= 48000. Lets save some memory: */ switch (rate) { case 44100: use_packsize = 36; break; case 48000: use_packsize = 42; break; case 88200: use_packsize = 72; break; } } if (!usb_stream_new(&us122l->sk, us122l->dev, 1, 2, rate, use_packsize, period_frames, 6)) goto out; err = us122l_set_sample_rate(us122l->dev, rate); if (err < 0) { us122l_stop(us122l); dev_err(&us122l->dev->dev, "us122l_set_sample_rate error\n"); goto out; } err = usb_stream_start(&us122l->sk); if (err < 0) { us122l_stop(us122l); dev_err(&us122l->dev->dev, "%s error %i\n", __func__, err); goto out; } list_for_each(p, &us122l->midi_list) snd_usbmidi_input_start(p); success = true; out: return success; } static int usb_stream_hwdep_ioctl(struct snd_hwdep *hw, struct file *file, unsigned int cmd, unsigned long arg) { struct usb_stream_config cfg; struct us122l *us122l = hw->private_data; struct usb_stream *s; unsigned int min_period_frames; int err = 0; bool high_speed; if (cmd != SNDRV_USB_STREAM_IOCTL_SET_PARAMS) return -ENOTTY; if (copy_from_user(&cfg, (void __user *)arg, sizeof(cfg))) return -EFAULT; if (cfg.version != USB_STREAM_INTERFACE_VERSION) return -ENXIO; high_speed = us122l->dev->speed == USB_SPEED_HIGH; if ((cfg.sample_rate != 44100 && cfg.sample_rate != 48000 && (!high_speed || (cfg.sample_rate != 88200 && cfg.sample_rate != 96000))) || cfg.frame_size != 6 || cfg.period_frames > 0x3000) return -EINVAL; switch (cfg.sample_rate) { case 44100: min_period_frames = 48; break; case 48000: min_period_frames = 52; break; default: min_period_frames = 104; break; } if (!high_speed) min_period_frames <<= 1; if (cfg.period_frames < min_period_frames) return -EINVAL; snd_power_wait(hw->card); mutex_lock(&us122l->mutex); s = us122l->sk.s; if (!us122l->master) { us122l->master = file; } else if (us122l->master != file) { if (!s || memcmp(&cfg, &s->cfg, sizeof(cfg))) { err = -EIO; goto unlock; } us122l->slave = file; } if (!s || memcmp(&cfg, &s->cfg, sizeof(cfg)) || s->state == usb_stream_xrun) { us122l_stop(us122l); if (!us122l_start(us122l, cfg.sample_rate, cfg.period_frames)) err = -EIO; else err = 1; } unlock: mutex_unlock(&us122l->mutex); wake_up_all(&us122l->sk.sleep); return err; } #define SND_USB_STREAM_ID "USB STREAM" static int usb_stream_hwdep_new(struct snd_card *card) { int err; struct snd_hwdep *hw; struct usb_device *dev = US122L(card)->dev; err = snd_hwdep_new(card, SND_USB_STREAM_ID, 0, &hw); if (err < 0) return err; hw->iface = SNDRV_HWDEP_IFACE_USB_STREAM; hw->private_data = US122L(card); hw->ops.open = usb_stream_hwdep_open; hw->ops.release = usb_stream_hwdep_release; hw->ops.ioctl = usb_stream_hwdep_ioctl; hw->ops.ioctl_compat = usb_stream_hwdep_ioctl; hw->ops.mmap = usb_stream_hwdep_mmap; hw->ops.poll = usb_stream_hwdep_poll; sprintf(hw->name, "/dev/bus/usb/%03d/%03d/hwdeppcm", dev->bus->busnum, dev->devnum); return 0; } static bool us122l_create_card(struct snd_card *card) { int err; struct us122l *us122l = US122L(card); if (us122l->is_us144) { err = usb_set_interface(us122l->dev, 0, 1); if (err) { dev_err(card->dev, "usb_set_interface error\n"); return false; } } err = usb_set_interface(us122l->dev, 1, 1); if (err) { dev_err(card->dev, "usb_set_interface error\n"); return false; } pt_info_set(us122l->dev, 0x11); pt_info_set(us122l->dev, 0x10); if (!us122l_start(us122l, 44100, 256)) return false; if (us122l->is_us144) err = us144_create_usbmidi(card); else err = us122l_create_usbmidi(card); if (err < 0) { dev_err(card->dev, "us122l_create_usbmidi error %i\n", err); goto stop; } err = usb_stream_hwdep_new(card); if (err < 0) { /* release the midi resources */ struct list_head *p; list_for_each(p, &us122l->midi_list) snd_usbmidi_disconnect(p); goto stop; } return true; stop: us122l_stop(us122l); return false; } static void snd_us122l_free(struct snd_card *card) { struct us122l *us122l = US122L(card); int index = us122l->card_index; if (index >= 0 && index < SNDRV_CARDS) snd_us122l_card_used[index] = 0; } static int usx2y_create_card(struct usb_device *device, struct usb_interface *intf, struct snd_card **cardp, unsigned long flags) { int dev; struct snd_card *card; int err; for (dev = 0; dev < SNDRV_CARDS; ++dev) if (enable[dev] && !snd_us122l_card_used[dev]) break; if (dev >= SNDRV_CARDS) return -ENODEV; err = snd_card_new(&intf->dev, index[dev], id[dev], THIS_MODULE, sizeof(struct us122l), &card); if (err < 0) return err; snd_us122l_card_used[US122L(card)->card_index = dev] = 1; card->private_free = snd_us122l_free; US122L(card)->dev = device; mutex_init(&US122L(card)->mutex); US122L(card)->sk.dev = device; init_waitqueue_head(&US122L(card)->sk.sleep); US122L(card)->is_us144 = flags & US122L_FLAG_US144; INIT_LIST_HEAD(&US122L(card)->midi_list); strcpy(card->driver, "USB "NAME_ALLCAPS""); sprintf(card->shortname, "TASCAM "NAME_ALLCAPS""); sprintf(card->longname, "%s (%x:%x if %d at %03d/%03d)", card->shortname, le16_to_cpu(device->descriptor.idVendor), le16_to_cpu(device->descriptor.idProduct), 0, US122L(card)->dev->bus->busnum, US122L(card)->dev->devnum ); *cardp = card; return 0; } static int us122l_usb_probe(struct usb_interface *intf, const struct usb_device_id *device_id, struct snd_card **cardp) { struct usb_device *device = interface_to_usbdev(intf); struct snd_card *card; int err; err = usx2y_create_card(device, intf, &card, device_id->driver_info); if (err < 0) return err; if (!us122l_create_card(card)) { snd_card_free(card); return -EINVAL; } err = snd_card_register(card); if (err < 0) { snd_card_free(card); return err; } usb_get_intf(usb_ifnum_to_if(device, 0)); usb_get_dev(device); *cardp = card; return 0; } static int snd_us122l_probe(struct usb_interface *intf, const struct usb_device_id *id) { struct usb_device *device = interface_to_usbdev(intf); struct snd_card *card; int err; if (id->driver_info & US122L_FLAG_US144 && device->speed == USB_SPEED_HIGH) { dev_err(&device->dev, "disable ehci-hcd to run US-144\n"); return -ENODEV; } if (intf->cur_altsetting->desc.bInterfaceNumber != 1) return 0; err = us122l_usb_probe(usb_get_intf(intf), id, &card); if (err < 0) { usb_put_intf(intf); return err; } usb_set_intfdata(intf, card); return 0; } static void snd_us122l_disconnect(struct usb_interface *intf) { struct snd_card *card; struct us122l *us122l; struct list_head *p; card = usb_get_intfdata(intf); if (!card) return; snd_card_disconnect(card); us122l = US122L(card); mutex_lock(&us122l->mutex); us122l_stop(us122l); mutex_unlock(&us122l->mutex); /* release the midi resources */ list_for_each(p, &us122l->midi_list) { snd_usbmidi_disconnect(p); } usb_put_intf(usb_ifnum_to_if(us122l->dev, 0)); usb_put_intf(usb_ifnum_to_if(us122l->dev, 1)); usb_put_dev(us122l->dev); snd_card_free_when_closed(card); } static int snd_us122l_suspend(struct usb_interface *intf, pm_message_t message) { struct snd_card *card; struct us122l *us122l; struct list_head *p; card = usb_get_intfdata(intf); if (!card) return 0; snd_power_change_state(card, SNDRV_CTL_POWER_D3hot); us122l = US122L(card); if (!us122l) return 0; list_for_each(p, &us122l->midi_list) snd_usbmidi_input_stop(p); mutex_lock(&us122l->mutex); usb_stream_stop(&us122l->sk); mutex_unlock(&us122l->mutex); return 0; } static int snd_us122l_resume(struct usb_interface *intf) { struct snd_card *card; struct us122l *us122l; struct list_head *p; int err; card = usb_get_intfdata(intf); if (!card) return 0; us122l = US122L(card); if (!us122l) return 0; mutex_lock(&us122l->mutex); /* needed, doesn't restart without: */ if (us122l->is_us144) { err = usb_set_interface(us122l->dev, 0, 1); if (err) { dev_err(&us122l->dev->dev, "usb_set_interface error\n"); goto unlock; } } err = usb_set_interface(us122l->dev, 1, 1); if (err) { dev_err(&us122l->dev->dev, "usb_set_interface error\n"); goto unlock; } pt_info_set(us122l->dev, 0x11); pt_info_set(us122l->dev, 0x10); err = us122l_set_sample_rate(us122l->dev, us122l->sk.s->cfg.sample_rate); if (err < 0) { dev_err(&us122l->dev->dev, "us122l_set_sample_rate error\n"); goto unlock; } err = usb_stream_start(&us122l->sk); if (err) goto unlock; list_for_each(p, &us122l->midi_list) snd_usbmidi_input_start(p); unlock: mutex_unlock(&us122l->mutex); snd_power_change_state(card, SNDRV_CTL_POWER_D0); return err; } static const struct usb_device_id snd_us122l_usb_id_table[] = { { .match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = 0x0644, .idProduct = USB_ID_US122L }, { /* US-144 only works at USB1.1! Disable module ehci-hcd. */ .match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = 0x0644, .idProduct = USB_ID_US144, .driver_info = US122L_FLAG_US144 }, { .match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = 0x0644, .idProduct = USB_ID_US122MKII }, { .match_flags = USB_DEVICE_ID_MATCH_DEVICE, .idVendor = 0x0644, .idProduct = USB_ID_US144MKII, .driver_info = US122L_FLAG_US144 }, { /* terminator */ } }; MODULE_DEVICE_TABLE(usb, snd_us122l_usb_id_table); static struct usb_driver snd_us122l_usb_driver = { .name = "snd-usb-us122l", .probe = snd_us122l_probe, .disconnect = snd_us122l_disconnect, .suspend = snd_us122l_suspend, .resume = snd_us122l_resume, .reset_resume = snd_us122l_resume, .id_table = snd_us122l_usb_id_table, .supports_autosuspend = 1 }; module_usb_driver(snd_us122l_usb_driver); |
| 71 2 69 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | // SPDX-License-Identifier: GPL-2.0 /* * fs/partitions/sysv68.c * * Copyright (C) 2007 Philippe De Muyter <phdm@macqel.be> */ #include "check.h" /* * Volume ID structure: on first 256-bytes sector of disk */ struct volumeid { u8 vid_unused[248]; u8 vid_mac[8]; /* ASCII string "MOTOROLA" */ }; /* * config block: second 256-bytes sector on disk */ struct dkconfig { u8 ios_unused0[128]; __be32 ios_slcblk; /* Slice table block number */ __be16 ios_slccnt; /* Number of entries in slice table */ u8 ios_unused1[122]; }; /* * combined volumeid and dkconfig block */ struct dkblk0 { struct volumeid dk_vid; struct dkconfig dk_ios; }; /* * Slice Table Structure */ struct slice { __be32 nblocks; /* slice size (in blocks) */ __be32 blkoff; /* block offset of slice */ }; int sysv68_partition(struct parsed_partitions *state) { int i, slices; int slot = 1; Sector sect; unsigned char *data; struct dkblk0 *b; struct slice *slice; char tmp[64]; data = read_part_sector(state, 0, §); if (!data) return -1; b = (struct dkblk0 *)data; if (memcmp(b->dk_vid.vid_mac, "MOTOROLA", sizeof(b->dk_vid.vid_mac))) { put_dev_sector(sect); return 0; } slices = be16_to_cpu(b->dk_ios.ios_slccnt); i = be32_to_cpu(b->dk_ios.ios_slcblk); put_dev_sector(sect); data = read_part_sector(state, i, §); if (!data) return -1; slices -= 1; /* last slice is the whole disk */ snprintf(tmp, sizeof(tmp), "sysV68: %s(s%u)", state->name, slices); strlcat(state->pp_buf, tmp, PAGE_SIZE); slice = (struct slice *)data; for (i = 0; i < slices; i++, slice++) { if (slot == state->limit) break; if (be32_to_cpu(slice->nblocks)) { put_partition(state, slot, be32_to_cpu(slice->blkoff), be32_to_cpu(slice->nblocks)); snprintf(tmp, sizeof(tmp), "(s%u)", i); strlcat(state->pp_buf, tmp, PAGE_SIZE); } slot++; } strlcat(state->pp_buf, "\n", PAGE_SIZE); put_dev_sector(sect); return 1; } |
| 9 9 1 5 1 1 5 5 5 5 25 25 10 10 1 1 1 1 4 4 1 1 2 9 1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 | // SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2020 Google Corporation */ #include <net/bluetooth/bluetooth.h> #include <net/bluetooth/hci_core.h> #include <net/bluetooth/mgmt.h> #include "mgmt_util.h" #include "msft.h" #define MSFT_RSSI_THRESHOLD_VALUE_MIN -127 #define MSFT_RSSI_THRESHOLD_VALUE_MAX 20 #define MSFT_RSSI_LOW_TIMEOUT_MAX 0x3C #define MSFT_OP_READ_SUPPORTED_FEATURES 0x00 struct msft_cp_read_supported_features { __u8 sub_opcode; } __packed; struct msft_rp_read_supported_features { __u8 status; __u8 sub_opcode; __le64 features; __u8 evt_prefix_len; __u8 evt_prefix[]; } __packed; #define MSFT_OP_LE_MONITOR_ADVERTISEMENT 0x03 #define MSFT_MONITOR_ADVERTISEMENT_TYPE_PATTERN 0x01 struct msft_le_monitor_advertisement_pattern { __u8 length; __u8 data_type; __u8 start_byte; __u8 pattern[]; }; struct msft_le_monitor_advertisement_pattern_data { __u8 count; __u8 data[]; }; struct msft_cp_le_monitor_advertisement { __u8 sub_opcode; __s8 rssi_high; __s8 rssi_low; __u8 rssi_low_interval; __u8 rssi_sampling_period; __u8 cond_type; __u8 data[]; } __packed; struct msft_rp_le_monitor_advertisement { __u8 status; __u8 sub_opcode; __u8 handle; } __packed; #define MSFT_OP_LE_CANCEL_MONITOR_ADVERTISEMENT 0x04 struct msft_cp_le_cancel_monitor_advertisement { __u8 sub_opcode; __u8 handle; } __packed; struct msft_rp_le_cancel_monitor_advertisement { __u8 status; __u8 sub_opcode; } __packed; #define MSFT_OP_LE_SET_ADVERTISEMENT_FILTER_ENABLE 0x05 struct msft_cp_le_set_advertisement_filter_enable { __u8 sub_opcode; __u8 enable; } __packed; struct msft_rp_le_set_advertisement_filter_enable { __u8 status; __u8 sub_opcode; } __packed; #define MSFT_EV_LE_MONITOR_DEVICE 0x02 struct msft_ev_le_monitor_device { __u8 addr_type; bdaddr_t bdaddr; __u8 monitor_handle; __u8 monitor_state; } __packed; struct msft_monitor_advertisement_handle_data { __u8 msft_handle; __u16 mgmt_handle; __s8 rssi_high; __s8 rssi_low; __u8 rssi_low_interval; __u8 rssi_sampling_period; __u8 cond_type; struct list_head list; }; enum monitor_addr_filter_state { AF_STATE_IDLE, AF_STATE_ADDING, AF_STATE_ADDED, AF_STATE_REMOVING, }; #define MSFT_MONITOR_ADVERTISEMENT_TYPE_ADDR 0x04 struct msft_monitor_addr_filter_data { __u8 msft_handle; __u8 pattern_handle; /* address filters pertain to */ __u16 mgmt_handle; int state; __s8 rssi_high; __s8 rssi_low; __u8 rssi_low_interval; __u8 rssi_sampling_period; __u8 addr_type; bdaddr_t bdaddr; struct list_head list; }; struct msft_data { __u64 features; __u8 evt_prefix_len; __u8 *evt_prefix; struct list_head handle_map; struct list_head address_filters; __u8 resuming; __u8 suspending; __u8 filter_enabled; /* To synchronize add/remove address filter and monitor device event.*/ struct mutex filter_lock; }; bool msft_monitor_supported(struct hci_dev *hdev) { return !!(msft_get_features(hdev) & MSFT_FEATURE_MASK_LE_ADV_MONITOR); } static bool read_supported_features(struct hci_dev *hdev, struct msft_data *msft) { struct msft_cp_read_supported_features cp; struct msft_rp_read_supported_features *rp; struct sk_buff *skb; cp.sub_opcode = MSFT_OP_READ_SUPPORTED_FEATURES; skb = __hci_cmd_sync(hdev, hdev->msft_opcode, sizeof(cp), &cp, HCI_CMD_TIMEOUT); if (IS_ERR(skb)) { bt_dev_err(hdev, "Failed to read MSFT supported features (%ld)", PTR_ERR(skb)); return false; } if (skb->len < sizeof(*rp)) { bt_dev_err(hdev, "MSFT supported features length mismatch"); goto failed; } rp = (struct msft_rp_read_supported_features *)skb->data; if (rp->sub_opcode != MSFT_OP_READ_SUPPORTED_FEATURES) goto failed; if (rp->evt_prefix_len > 0) { msft->evt_prefix = kmemdup(rp->evt_prefix, rp->evt_prefix_len, GFP_KERNEL); if (!msft->evt_prefix) goto failed; } msft->evt_prefix_len = rp->evt_prefix_len; msft->features = __le64_to_cpu(rp->features); if (msft->features & MSFT_FEATURE_MASK_CURVE_VALIDITY) hdev->msft_curve_validity = true; kfree_skb(skb); return true; failed: kfree_skb(skb); return false; } /* is_mgmt = true matches the handle exposed to userspace via mgmt. * is_mgmt = false matches the handle used by the msft controller. * This function requires the caller holds hdev->lock */ static struct msft_monitor_advertisement_handle_data *msft_find_handle_data (struct hci_dev *hdev, u16 handle, bool is_mgmt) { struct msft_monitor_advertisement_handle_data *entry; struct msft_data *msft = hdev->msft_data; list_for_each_entry(entry, &msft->handle_map, list) { if (is_mgmt && entry->mgmt_handle == handle) return entry; if (!is_mgmt && entry->msft_handle == handle) return entry; } return NULL; } /* This function requires the caller holds msft->filter_lock */ static struct msft_monitor_addr_filter_data *msft_find_address_data (struct hci_dev *hdev, u8 addr_type, bdaddr_t *addr, u8 pattern_handle) { struct msft_monitor_addr_filter_data *entry; struct msft_data *msft = hdev->msft_data; list_for_each_entry(entry, &msft->address_filters, list) { if (entry->pattern_handle == pattern_handle && addr_type == entry->addr_type && !bacmp(addr, &entry->bdaddr)) return entry; } return NULL; } /* This function requires the caller holds hdev->lock */ static int msft_monitor_device_del(struct hci_dev *hdev, __u16 mgmt_handle, bdaddr_t *bdaddr, __u8 addr_type, bool notify) { struct monitored_device *dev, *tmp; int count = 0; list_for_each_entry_safe(dev, tmp, &hdev->monitored_devices, list) { /* mgmt_handle == 0 indicates remove all devices, whereas, * bdaddr == NULL indicates remove all devices matching the * mgmt_handle. */ if ((!mgmt_handle || dev->handle == mgmt_handle) && (!bdaddr || (!bacmp(bdaddr, &dev->bdaddr) && addr_type == dev->addr_type))) { if (notify && dev->notified) { mgmt_adv_monitor_device_lost(hdev, dev->handle, &dev->bdaddr, dev->addr_type); } list_del(&dev->list); kfree(dev); count++; } } return count; } static int msft_le_monitor_advertisement_cb(struct hci_dev *hdev, u16 opcode, struct adv_monitor *monitor, struct sk_buff *skb) { struct msft_rp_le_monitor_advertisement *rp; struct msft_monitor_advertisement_handle_data *handle_data; struct msft_data *msft = hdev->msft_data; int status = 0; hci_dev_lock(hdev); rp = (struct msft_rp_le_monitor_advertisement *)skb->data; if (skb->len < sizeof(*rp)) { status = HCI_ERROR_UNSPECIFIED; goto unlock; } status = rp->status; if (status) goto unlock; handle_data = kmalloc(sizeof(*handle_data), GFP_KERNEL); if (!handle_data) { status = HCI_ERROR_UNSPECIFIED; goto unlock; } handle_data->mgmt_handle = monitor->handle; handle_data->msft_handle = rp->handle; handle_data->cond_type = MSFT_MONITOR_ADVERTISEMENT_TYPE_PATTERN; INIT_LIST_HEAD(&handle_data->list); list_add(&handle_data->list, &msft->handle_map); monitor->state = ADV_MONITOR_STATE_OFFLOADED; unlock: if (status) hci_free_adv_monitor(hdev, monitor); hci_dev_unlock(hdev); return status; } /* This function requires the caller holds hci_req_sync_lock */ static void msft_remove_addr_filters_sync(struct hci_dev *hdev, u8 handle) { struct msft_monitor_addr_filter_data *address_filter, *n; struct msft_cp_le_cancel_monitor_advertisement cp; struct msft_data *msft = hdev->msft_data; struct list_head head; struct sk_buff *skb; INIT_LIST_HEAD(&head); /* Cancel all corresponding address monitors */ mutex_lock(&msft->filter_lock); list_for_each_entry_safe(address_filter, n, &msft->address_filters, list) { if (address_filter->pattern_handle != handle) continue; list_del(&address_filter->list); /* Keep the address filter and let * msft_add_address_filter_sync() remove and free the address * filter. */ if (address_filter->state == AF_STATE_ADDING) { address_filter->state = AF_STATE_REMOVING; continue; } /* Keep the address filter and let * msft_cancel_address_filter_sync() remove and free the address * filter */ if (address_filter->state == AF_STATE_REMOVING) continue; list_add_tail(&address_filter->list, &head); } mutex_unlock(&msft->filter_lock); list_for_each_entry_safe(address_filter, n, &head, list) { list_del(&address_filter->list); cp.sub_opcode = MSFT_OP_LE_CANCEL_MONITOR_ADVERTISEMENT; cp.handle = address_filter->msft_handle; skb = __hci_cmd_sync(hdev, hdev->msft_opcode, sizeof(cp), &cp, HCI_CMD_TIMEOUT); if (IS_ERR(skb)) { kfree(address_filter); continue; } kfree_skb(skb); bt_dev_dbg(hdev, "MSFT: Canceled device %pMR address filter", &address_filter->bdaddr); kfree(address_filter); } } static int msft_le_cancel_monitor_advertisement_cb(struct hci_dev *hdev, u16 opcode, struct adv_monitor *monitor, struct sk_buff *skb) { struct msft_rp_le_cancel_monitor_advertisement *rp; struct msft_monitor_advertisement_handle_data *handle_data; struct msft_data *msft = hdev->msft_data; int status = 0; u8 msft_handle; rp = (struct msft_rp_le_cancel_monitor_advertisement *)skb->data; if (skb->len < sizeof(*rp)) { status = HCI_ERROR_UNSPECIFIED; goto done; } status = rp->status; if (status) goto done; hci_dev_lock(hdev); handle_data = msft_find_handle_data(hdev, monitor->handle, true); if (handle_data) { if (monitor->state == ADV_MONITOR_STATE_OFFLOADED) monitor->state = ADV_MONITOR_STATE_REGISTERED; /* Do not free the monitor if it is being removed due to * suspend. It will be re-monitored on resume. */ if (!msft->suspending) { hci_free_adv_monitor(hdev, monitor); /* Clear any monitored devices by this Adv Monitor */ msft_monitor_device_del(hdev, handle_data->mgmt_handle, NULL, 0, false); } msft_handle = handle_data->msft_handle; list_del(&handle_data->list); kfree(handle_data); hci_dev_unlock(hdev); msft_remove_addr_filters_sync(hdev, msft_handle); } else { hci_dev_unlock(hdev); } done: return status; } /* This function requires the caller holds hci_req_sync_lock */ static int msft_remove_monitor_sync(struct hci_dev *hdev, struct adv_monitor *monitor) { struct msft_cp_le_cancel_monitor_advertisement cp; struct msft_monitor_advertisement_handle_data *handle_data; struct sk_buff *skb; handle_data = msft_find_handle_data(hdev, monitor->handle, true); /* If no matched handle, just remove without telling controller */ if (!handle_data) return -ENOENT; cp.sub_opcode = MSFT_OP_LE_CANCEL_MONITOR_ADVERTISEMENT; cp.handle = handle_data->msft_handle; skb = __hci_cmd_sync(hdev, hdev->msft_opcode, sizeof(cp), &cp, HCI_CMD_TIMEOUT); if (IS_ERR(skb)) return PTR_ERR(skb); return msft_le_cancel_monitor_advertisement_cb(hdev, hdev->msft_opcode, monitor, skb); } /* This function requires the caller holds hci_req_sync_lock */ int msft_suspend_sync(struct hci_dev *hdev) { struct msft_data *msft = hdev->msft_data; struct adv_monitor *monitor; int handle = 0; if (!msft || !msft_monitor_supported(hdev)) return 0; msft->suspending = true; while (1) { monitor = idr_get_next(&hdev->adv_monitors_idr, &handle); if (!monitor) break; msft_remove_monitor_sync(hdev, monitor); handle++; } /* All monitors have been removed */ msft->suspending = false; return 0; } static bool msft_monitor_rssi_valid(struct adv_monitor *monitor) { struct adv_rssi_thresholds *r = &monitor->rssi; if (r->high_threshold < MSFT_RSSI_THRESHOLD_VALUE_MIN || r->high_threshold > MSFT_RSSI_THRESHOLD_VALUE_MAX || r->low_threshold < MSFT_RSSI_THRESHOLD_VALUE_MIN || r->low_threshold > MSFT_RSSI_THRESHOLD_VALUE_MAX) return false; /* High_threshold_timeout is not supported, * once high_threshold is reached, events are immediately reported. */ if (r->high_threshold_timeout != 0) return false; if (r->low_threshold_timeout > MSFT_RSSI_LOW_TIMEOUT_MAX) return false; /* Sampling period from 0x00 to 0xFF are all allowed */ return true; } static bool msft_monitor_pattern_valid(struct adv_monitor *monitor) { return msft_monitor_rssi_valid(monitor); /* No additional check needed for pattern-based monitor */ } static int msft_add_monitor_sync(struct hci_dev *hdev, struct adv_monitor *monitor) { struct msft_cp_le_monitor_advertisement *cp; struct msft_le_monitor_advertisement_pattern_data *pattern_data; struct msft_monitor_advertisement_handle_data *handle_data; struct msft_le_monitor_advertisement_pattern *pattern; struct adv_pattern *entry; size_t total_size = sizeof(*cp) + sizeof(*pattern_data); ptrdiff_t offset = 0; u8 pattern_count = 0; struct sk_buff *skb; int err; if (!msft_monitor_pattern_valid(monitor)) return -EINVAL; list_for_each_entry(entry, &monitor->patterns, list) { pattern_count++; total_size += sizeof(*pattern) + entry->length; } cp = kmalloc(total_size, GFP_KERNEL); if (!cp) return -ENOMEM; cp->sub_opcode = MSFT_OP_LE_MONITOR_ADVERTISEMENT; cp->rssi_high = monitor->rssi.high_threshold; cp->rssi_low = monitor->rssi.low_threshold; cp->rssi_low_interval = (u8)monitor->rssi.low_threshold_timeout; cp->rssi_sampling_period = monitor->rssi.sampling_period; cp->cond_type = MSFT_MONITOR_ADVERTISEMENT_TYPE_PATTERN; pattern_data = (void *)cp->data; pattern_data->count = pattern_count; list_for_each_entry(entry, &monitor->patterns, list) { pattern = (void *)(pattern_data->data + offset); /* the length also includes data_type and offset */ pattern->length = entry->length + 2; pattern->data_type = entry->ad_type; pattern->start_byte = entry->offset; memcpy(pattern->pattern, entry->value, entry->length); offset += sizeof(*pattern) + entry->length; } skb = __hci_cmd_sync(hdev, hdev->msft_opcode, total_size, cp, HCI_CMD_TIMEOUT); if (IS_ERR(skb)) { err = PTR_ERR(skb); goto out_free; } err = msft_le_monitor_advertisement_cb(hdev, hdev->msft_opcode, monitor, skb); if (err) goto out_free; handle_data = msft_find_handle_data(hdev, monitor->handle, true); if (!handle_data) { err = -ENODATA; goto out_free; } handle_data->rssi_high = cp->rssi_high; handle_data->rssi_low = cp->rssi_low; handle_data->rssi_low_interval = cp->rssi_low_interval; handle_data->rssi_sampling_period = cp->rssi_sampling_period; out_free: kfree(cp); return err; } /* This function requires the caller holds hci_req_sync_lock */ static void reregister_monitor(struct hci_dev *hdev) { struct adv_monitor *monitor; struct msft_data *msft = hdev->msft_data; int handle = 0; if (!msft) return; msft->resuming = true; while (1) { monitor = idr_get_next(&hdev->adv_monitors_idr, &handle); if (!monitor) break; msft_add_monitor_sync(hdev, monitor); handle++; } /* All monitors have been reregistered */ msft->resuming = false; } /* This function requires the caller holds hci_req_sync_lock */ int msft_resume_sync(struct hci_dev *hdev) { struct msft_data *msft = hdev->msft_data; if (!msft || !msft_monitor_supported(hdev)) return 0; hci_dev_lock(hdev); /* Clear already tracked devices on resume. Once the monitors are * reregistered, devices in range will be found again after resume. */ hdev->advmon_pend_notify = false; msft_monitor_device_del(hdev, 0, NULL, 0, true); hci_dev_unlock(hdev); reregister_monitor(hdev); return 0; } /* This function requires the caller holds hci_req_sync_lock */ void msft_do_open(struct hci_dev *hdev) { struct msft_data *msft = hdev->msft_data; if (hdev->msft_opcode == HCI_OP_NOP) return; if (!msft) { bt_dev_err(hdev, "MSFT extension not registered"); return; } bt_dev_dbg(hdev, "Initialize MSFT extension"); /* Reset existing MSFT data before re-reading */ kfree(msft->evt_prefix); msft->evt_prefix = NULL; msft->evt_prefix_len = 0; msft->features = 0; if (!read_supported_features(hdev, msft)) { hdev->msft_data = NULL; kfree(msft); return; } if (msft_monitor_supported(hdev)) { msft->resuming = true; msft_set_filter_enable(hdev, true); /* Monitors get removed on power off, so we need to explicitly * tell the controller to re-monitor. */ reregister_monitor(hdev); } } void msft_do_close(struct hci_dev *hdev) { struct msft_data *msft = hdev->msft_data; struct msft_monitor_advertisement_handle_data *handle_data, *tmp; struct msft_monitor_addr_filter_data *address_filter, *n; struct adv_monitor *monitor; if (!msft) return; bt_dev_dbg(hdev, "Cleanup of MSFT extension"); /* The controller will silently remove all monitors on power off. * Therefore, remove handle_data mapping and reset monitor state. */ list_for_each_entry_safe(handle_data, tmp, &msft->handle_map, list) { monitor = idr_find(&hdev->adv_monitors_idr, handle_data->mgmt_handle); if (monitor && monitor->state == ADV_MONITOR_STATE_OFFLOADED) monitor->state = ADV_MONITOR_STATE_REGISTERED; list_del(&handle_data->list); kfree(handle_data); } mutex_lock(&msft->filter_lock); list_for_each_entry_safe(address_filter, n, &msft->address_filters, list) { list_del(&address_filter->list); kfree(address_filter); } mutex_unlock(&msft->filter_lock); hci_dev_lock(hdev); /* Clear any devices that are being monitored and notify device lost */ hdev->advmon_pend_notify = false; msft_monitor_device_del(hdev, 0, NULL, 0, true); hci_dev_unlock(hdev); } static int msft_cancel_address_filter_sync(struct hci_dev *hdev, void *data) { struct msft_monitor_addr_filter_data *address_filter = data; struct msft_cp_le_cancel_monitor_advertisement cp; struct msft_data *msft = hdev->msft_data; struct sk_buff *skb; int err = 0; if (!msft) { bt_dev_err(hdev, "MSFT: msft data is freed"); return -EINVAL; } /* The address filter has been removed by hci dev close */ if (!test_bit(HCI_UP, &hdev->flags)) return 0; mutex_lock(&msft->filter_lock); list_del(&address_filter->list); mutex_unlock(&msft->filter_lock); cp.sub_opcode = MSFT_OP_LE_CANCEL_MONITOR_ADVERTISEMENT; cp.handle = address_filter->msft_handle; skb = __hci_cmd_sync(hdev, hdev->msft_opcode, sizeof(cp), &cp, HCI_CMD_TIMEOUT); if (IS_ERR(skb)) { bt_dev_err(hdev, "MSFT: Failed to cancel address (%pMR) filter", &address_filter->bdaddr); err = PTR_ERR(skb); goto done; } kfree_skb(skb); bt_dev_dbg(hdev, "MSFT: Canceled device %pMR address filter", &address_filter->bdaddr); done: kfree(address_filter); return err; } void msft_register(struct hci_dev *hdev) { struct msft_data *msft = NULL; bt_dev_dbg(hdev, "Register MSFT extension"); msft = kzalloc(sizeof(*msft), GFP_KERNEL); if (!msft) { bt_dev_err(hdev, "Failed to register MSFT extension"); return; } INIT_LIST_HEAD(&msft->handle_map); INIT_LIST_HEAD(&msft->address_filters); hdev->msft_data = msft; mutex_init(&msft->filter_lock); } void msft_release(struct hci_dev *hdev) { struct msft_data *msft = hdev->msft_data; if (!msft) return; bt_dev_dbg(hdev, "Unregister MSFT extension"); hdev->msft_data = NULL; kfree(msft->evt_prefix); mutex_destroy(&msft->filter_lock); kfree(msft); } /* This function requires the caller holds hdev->lock */ static void msft_device_found(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 addr_type, __u16 mgmt_handle) { struct monitored_device *dev; dev = kmalloc(sizeof(*dev), GFP_KERNEL); if (!dev) { bt_dev_err(hdev, "MSFT vendor event %u: no memory", MSFT_EV_LE_MONITOR_DEVICE); return; } bacpy(&dev->bdaddr, bdaddr); dev->addr_type = addr_type; dev->handle = mgmt_handle; dev->notified = false; INIT_LIST_HEAD(&dev->list); list_add(&dev->list, &hdev->monitored_devices); hdev->advmon_pend_notify = true; } /* This function requires the caller holds hdev->lock */ static void msft_device_lost(struct hci_dev *hdev, bdaddr_t *bdaddr, __u8 addr_type, __u16 mgmt_handle) { if (!msft_monitor_device_del(hdev, mgmt_handle, bdaddr, addr_type, true)) { bt_dev_err(hdev, "MSFT vendor event %u: dev %pMR not in list", MSFT_EV_LE_MONITOR_DEVICE, bdaddr); } } static void *msft_skb_pull(struct hci_dev *hdev, struct sk_buff *skb, u8 ev, size_t len) { void *data; data = skb_pull_data(skb, len); if (!data) bt_dev_err(hdev, "Malformed MSFT vendor event: 0x%02x", ev); return data; } static int msft_add_address_filter_sync(struct hci_dev *hdev, void *data) { struct msft_monitor_addr_filter_data *address_filter = data; struct msft_rp_le_monitor_advertisement *rp; struct msft_cp_le_monitor_advertisement *cp; struct msft_data *msft = hdev->msft_data; struct sk_buff *skb = NULL; bool remove = false; size_t size; if (!msft) { bt_dev_err(hdev, "MSFT: msft data is freed"); return -EINVAL; } /* The address filter has been removed by hci dev close */ if (!test_bit(HCI_UP, &hdev->flags)) return -ENODEV; /* We are safe to use the address filter from now on. * msft_monitor_device_evt() wouldn't delete this filter because it's * not been added by now. * And all other functions that requiring hci_req_sync_lock wouldn't * touch this filter before this func completes because it's protected * by hci_req_sync_lock. */ if (address_filter->state == AF_STATE_REMOVING) { mutex_lock(&msft->filter_lock); list_del(&address_filter->list); mutex_unlock(&msft->filter_lock); kfree(address_filter); return 0; } size = sizeof(*cp) + sizeof(address_filter->addr_type) + sizeof(address_filter->bdaddr); cp = kzalloc(size, GFP_KERNEL); if (!cp) { bt_dev_err(hdev, "MSFT: Alloc cmd param err"); remove = true; goto done; } cp->sub_opcode = MSFT_OP_LE_MONITOR_ADVERTISEMENT; cp->rssi_high = address_filter->rssi_high; cp->rssi_low = address_filter->rssi_low; cp->rssi_low_interval = address_filter->rssi_low_interval; cp->rssi_sampling_period = address_filter->rssi_sampling_period; cp->cond_type = MSFT_MONITOR_ADVERTISEMENT_TYPE_ADDR; cp->data[0] = address_filter->addr_type; memcpy(&cp->data[1], &address_filter->bdaddr, sizeof(address_filter->bdaddr)); skb = __hci_cmd_sync(hdev, hdev->msft_opcode, size, cp, HCI_CMD_TIMEOUT); kfree(cp); if (IS_ERR(skb)) { bt_dev_err(hdev, "Failed to enable address %pMR filter", &address_filter->bdaddr); skb = NULL; remove = true; goto done; } rp = skb_pull_data(skb, sizeof(*rp)); if (!rp || rp->sub_opcode != MSFT_OP_LE_MONITOR_ADVERTISEMENT || rp->status) remove = true; done: mutex_lock(&msft->filter_lock); if (remove) { bt_dev_warn(hdev, "MSFT: Remove address (%pMR) filter", &address_filter->bdaddr); list_del(&address_filter->list); kfree(address_filter); } else { address_filter->state = AF_STATE_ADDED; address_filter->msft_handle = rp->handle; bt_dev_dbg(hdev, "MSFT: Address %pMR filter enabled", &address_filter->bdaddr); } mutex_unlock(&msft->filter_lock); kfree_skb(skb); return 0; } /* This function requires the caller holds msft->filter_lock */ static struct msft_monitor_addr_filter_data *msft_add_address_filter (struct hci_dev *hdev, u8 addr_type, bdaddr_t *bdaddr, struct msft_monitor_advertisement_handle_data *handle_data) { struct msft_monitor_addr_filter_data *address_filter = NULL; struct msft_data *msft = hdev->msft_data; int err; address_filter = kzalloc(sizeof(*address_filter), GFP_KERNEL); if (!address_filter) return NULL; address_filter->state = AF_STATE_ADDING; address_filter->msft_handle = 0xff; address_filter->pattern_handle = handle_data->msft_handle; address_filter->mgmt_handle = handle_data->mgmt_handle; address_filter->rssi_high = handle_data->rssi_high; address_filter->rssi_low = handle_data->rssi_low; address_filter->rssi_low_interval = handle_data->rssi_low_interval; address_filter->rssi_sampling_period = handle_data->rssi_sampling_period; address_filter->addr_type = addr_type; bacpy(&address_filter->bdaddr, bdaddr); /* With the above AF_STATE_ADDING, duplicated address filter can be * avoided when receiving monitor device event (found/lost) frequently * for the same device. */ list_add_tail(&address_filter->list, &msft->address_filters); err = hci_cmd_sync_queue(hdev, msft_add_address_filter_sync, address_filter, NULL); if (err < 0) { bt_dev_err(hdev, "MSFT: Add address %pMR filter err", bdaddr); list_del(&address_filter->list); kfree(address_filter); return NULL; } bt_dev_dbg(hdev, "MSFT: Add device %pMR address filter", &address_filter->bdaddr); return address_filter; } /* This function requires the caller holds hdev->lock */ static void msft_monitor_device_evt(struct hci_dev *hdev, struct sk_buff *skb) { struct msft_monitor_addr_filter_data *n, *address_filter = NULL; struct msft_ev_le_monitor_device *ev; struct msft_monitor_advertisement_handle_data *handle_data; struct msft_data *msft = hdev->msft_data; u16 mgmt_handle = 0xffff; u8 addr_type; ev = msft_skb_pull(hdev, skb, MSFT_EV_LE_MONITOR_DEVICE, sizeof(*ev)); if (!ev) return; bt_dev_dbg(hdev, "MSFT vendor event 0x%02x: handle 0x%04x state %d addr %pMR", MSFT_EV_LE_MONITOR_DEVICE, ev->monitor_handle, ev->monitor_state, &ev->bdaddr); handle_data = msft_find_handle_data(hdev, ev->monitor_handle, false); if (!test_bit(HCI_QUIRK_USE_MSFT_EXT_ADDRESS_FILTER, &hdev->quirks)) { if (!handle_data) return; mgmt_handle = handle_data->mgmt_handle; goto report_state; } if (handle_data) { /* Don't report any device found/lost event from pattern * monitors. Pattern monitor always has its address filters for * tracking devices. */ address_filter = msft_find_address_data(hdev, ev->addr_type, &ev->bdaddr, handle_data->msft_handle); if (address_filter) return; if (ev->monitor_state && handle_data->cond_type == MSFT_MONITOR_ADVERTISEMENT_TYPE_PATTERN) msft_add_address_filter(hdev, ev->addr_type, &ev->bdaddr, handle_data); return; } /* This device event is not from pattern monitor. * Report it if there is a corresponding address_filter for it. */ list_for_each_entry(n, &msft->address_filters, list) { if (n->state == AF_STATE_ADDED && n->msft_handle == ev->monitor_handle) { mgmt_handle = n->mgmt_handle; address_filter = n; break; } } if (!address_filter) { bt_dev_warn(hdev, "MSFT: Unexpected device event %pMR, %u, %u", &ev->bdaddr, ev->monitor_handle, ev->monitor_state); return; } report_state: switch (ev->addr_type) { case ADDR_LE_DEV_PUBLIC: addr_type = BDADDR_LE_PUBLIC; break; case ADDR_LE_DEV_RANDOM: addr_type = BDADDR_LE_RANDOM; break; default: bt_dev_err(hdev, "MSFT vendor event 0x%02x: unknown addr type 0x%02x", MSFT_EV_LE_MONITOR_DEVICE, ev->addr_type); return; } if (ev->monitor_state) { msft_device_found(hdev, &ev->bdaddr, addr_type, mgmt_handle); } else { if (address_filter && address_filter->state == AF_STATE_ADDED) { address_filter->state = AF_STATE_REMOVING; hci_cmd_sync_queue(hdev, msft_cancel_address_filter_sync, address_filter, NULL); } msft_device_lost(hdev, &ev->bdaddr, addr_type, mgmt_handle); } } void msft_vendor_evt(struct hci_dev *hdev, void *data, struct sk_buff *skb) { struct msft_data *msft = hdev->msft_data; u8 *evt_prefix; u8 *evt; if (!msft) return; /* When the extension has defined an event prefix, check that it * matches, and otherwise just return. */ if (msft->evt_prefix_len > 0) { evt_prefix = msft_skb_pull(hdev, skb, 0, msft->evt_prefix_len); if (!evt_prefix) return; if (memcmp(evt_prefix, msft->evt_prefix, msft->evt_prefix_len)) return; } /* Every event starts at least with an event code and the rest of * the data is variable and depends on the event code. */ if (skb->len < 1) return; evt = msft_skb_pull(hdev, skb, 0, sizeof(*evt)); if (!evt) return; hci_dev_lock(hdev); switch (*evt) { case MSFT_EV_LE_MONITOR_DEVICE: mutex_lock(&msft->filter_lock); msft_monitor_device_evt(hdev, skb); mutex_unlock(&msft->filter_lock); break; default: bt_dev_dbg(hdev, "MSFT vendor event 0x%02x", *evt); break; } hci_dev_unlock(hdev); } __u64 msft_get_features(struct hci_dev *hdev) { struct msft_data *msft = hdev->msft_data; return msft ? msft->features : 0; } static void msft_le_set_advertisement_filter_enable_cb(struct hci_dev *hdev, void *user_data, u8 status) { struct msft_cp_le_set_advertisement_filter_enable *cp = user_data; struct msft_data *msft = hdev->msft_data; /* Error 0x0C would be returned if the filter enabled status is * already set to whatever we were trying to set. * Although the default state should be disabled, some controller set * the initial value to enabled. Because there is no way to know the * actual initial value before sending this command, here we also treat * error 0x0C as success. */ if (status != 0x00 && status != 0x0C) return; hci_dev_lock(hdev); msft->filter_enabled = cp->enable; if (status == 0x0C) bt_dev_warn(hdev, "MSFT filter_enable is already %s", cp->enable ? "on" : "off"); hci_dev_unlock(hdev); } /* This function requires the caller holds hci_req_sync_lock */ int msft_add_monitor_pattern(struct hci_dev *hdev, struct adv_monitor *monitor) { struct msft_data *msft = hdev->msft_data; if (!msft) return -EOPNOTSUPP; if (msft->resuming || msft->suspending) return -EBUSY; return msft_add_monitor_sync(hdev, monitor); } /* This function requires the caller holds hci_req_sync_lock */ int msft_remove_monitor(struct hci_dev *hdev, struct adv_monitor *monitor) { struct msft_data *msft = hdev->msft_data; if (!msft) return -EOPNOTSUPP; if (msft->resuming || msft->suspending) return -EBUSY; return msft_remove_monitor_sync(hdev, monitor); } int msft_set_filter_enable(struct hci_dev *hdev, bool enable) { struct msft_cp_le_set_advertisement_filter_enable cp; struct msft_data *msft = hdev->msft_data; int err; if (!msft) return -EOPNOTSUPP; cp.sub_opcode = MSFT_OP_LE_SET_ADVERTISEMENT_FILTER_ENABLE; cp.enable = enable; err = __hci_cmd_sync_status(hdev, hdev->msft_opcode, sizeof(cp), &cp, HCI_CMD_TIMEOUT); msft_le_set_advertisement_filter_enable_cb(hdev, &cp, err); return 0; } bool msft_curve_validity(struct hci_dev *hdev) { return hdev->msft_curve_validity; } |
| 16 7 1 163 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 | /* SPDX-License-Identifier: GPL-2.0-or-later */ /* * Hash: Hash algorithms under the crypto API * * Copyright (c) 2008 Herbert Xu <herbert@gondor.apana.org.au> */ #ifndef _CRYPTO_HASH_H #define _CRYPTO_HASH_H #include <linux/atomic.h> #include <linux/crypto.h> #include <linux/string.h> struct crypto_ahash; /** * DOC: Message Digest Algorithm Definitions * * These data structures define modular message digest algorithm * implementations, managed via crypto_register_ahash(), * crypto_register_shash(), crypto_unregister_ahash() and * crypto_unregister_shash(). */ /* * struct hash_alg_common - define properties of message digest * @digestsize: Size of the result of the transformation. A buffer of this size * must be available to the @final and @finup calls, so they can * store the resulting hash into it. For various predefined sizes, * search include/crypto/ using * git grep _DIGEST_SIZE include/crypto. * @statesize: Size of the block for partial state of the transformation. A * buffer of this size must be passed to the @export function as it * will save the partial state of the transformation into it. On the * other side, the @import function will load the state from a * buffer of this size as well. * @base: Start of data structure of cipher algorithm. The common data * structure of crypto_alg contains information common to all ciphers. * The hash_alg_common data structure now adds the hash-specific * information. */ #define HASH_ALG_COMMON { \ unsigned int digestsize; \ unsigned int statesize; \ \ struct crypto_alg base; \ } struct hash_alg_common HASH_ALG_COMMON; struct ahash_request { struct crypto_async_request base; unsigned int nbytes; struct scatterlist *src; u8 *result; /* This field may only be used by the ahash API code. */ void *priv; void *__ctx[] CRYPTO_MINALIGN_ATTR; }; /** * struct ahash_alg - asynchronous message digest definition * @init: **[mandatory]** Initialize the transformation context. Intended only to initialize the * state of the HASH transformation at the beginning. This shall fill in * the internal structures used during the entire duration of the whole * transformation. No data processing happens at this point. Driver code * implementation must not use req->result. * @update: **[mandatory]** Push a chunk of data into the driver for transformation. This * function actually pushes blocks of data from upper layers into the * driver, which then passes those to the hardware as seen fit. This * function must not finalize the HASH transformation by calculating the * final message digest as this only adds more data into the * transformation. This function shall not modify the transformation * context, as this function may be called in parallel with the same * transformation object. Data processing can happen synchronously * [SHASH] or asynchronously [AHASH] at this point. Driver must not use * req->result. * @final: **[mandatory]** Retrieve result from the driver. This function finalizes the * transformation and retrieves the resulting hash from the driver and * pushes it back to upper layers. No data processing happens at this * point unless hardware requires it to finish the transformation * (then the data buffered by the device driver is processed). * @finup: **[optional]** Combination of @update and @final. This function is effectively a * combination of @update and @final calls issued in sequence. As some * hardware cannot do @update and @final separately, this callback was * added to allow such hardware to be used at least by IPsec. Data * processing can happen synchronously [SHASH] or asynchronously [AHASH] * at this point. * @digest: Combination of @init and @update and @final. This function * effectively behaves as the entire chain of operations, @init, * @update and @final issued in sequence. Just like @finup, this was * added for hardware which cannot do even the @finup, but can only do * the whole transformation in one run. Data processing can happen * synchronously [SHASH] or asynchronously [AHASH] at this point. * @setkey: Set optional key used by the hashing algorithm. Intended to push * optional key used by the hashing algorithm from upper layers into * the driver. This function can store the key in the transformation * context or can outright program it into the hardware. In the former * case, one must be careful to program the key into the hardware at * appropriate time and one must be careful that .setkey() can be * called multiple times during the existence of the transformation * object. Not all hashing algorithms do implement this function as it * is only needed for keyed message digests. SHAx/MDx/CRCx do NOT * implement this function. HMAC(MDx)/HMAC(SHAx)/CMAC(AES) do implement * this function. This function must be called before any other of the * @init, @update, @final, @finup, @digest is called. No data * processing happens at this point. * @export: Export partial state of the transformation. This function dumps the * entire state of the ongoing transformation into a provided block of * data so it can be @import 'ed back later on. This is useful in case * you want to save partial result of the transformation after * processing certain amount of data and reload this partial result * multiple times later on for multiple re-use. No data processing * happens at this point. Driver must not use req->result. * @import: Import partial state of the transformation. This function loads the * entire state of the ongoing transformation from a provided block of * data so the transformation can continue from this point onward. No * data processing happens at this point. Driver must not use * req->result. * @init_tfm: Initialize the cryptographic transformation object. * This function is called only once at the instantiation * time, right after the transformation context was * allocated. In case the cryptographic hardware has * some special requirements which need to be handled * by software, this function shall check for the precise * requirement of the transformation and put any software * fallbacks in place. * @exit_tfm: Deinitialize the cryptographic transformation object. * This is a counterpart to @init_tfm, used to remove * various changes set in @init_tfm. * @clone_tfm: Copy transform into new object, may allocate memory. * @halg: see struct hash_alg_common */ struct ahash_alg { int (*init)(struct ahash_request *req); int (*update)(struct ahash_request *req); int (*final)(struct ahash_request *req); int (*finup)(struct ahash_request *req); int (*digest)(struct ahash_request *req); int (*export)(struct ahash_request *req, void *out); int (*import)(struct ahash_request *req, const void *in); int (*setkey)(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen); int (*init_tfm)(struct crypto_ahash *tfm); void (*exit_tfm)(struct crypto_ahash *tfm); int (*clone_tfm)(struct crypto_ahash *dst, struct crypto_ahash *src); struct hash_alg_common halg; }; struct shash_desc { struct crypto_shash *tfm; void *__ctx[] __aligned(ARCH_SLAB_MINALIGN); }; #define HASH_MAX_DIGESTSIZE 64 /* * Worst case is hmac(sha3-224-generic). Its context is a nested 'shash_desc' * containing a 'struct sha3_state'. */ #define HASH_MAX_DESCSIZE (sizeof(struct shash_desc) + 360) #define SHASH_DESC_ON_STACK(shash, ctx) \ char __##shash##_desc[sizeof(struct shash_desc) + HASH_MAX_DESCSIZE] \ __aligned(__alignof__(struct shash_desc)); \ struct shash_desc *shash = (struct shash_desc *)__##shash##_desc /** * struct shash_alg - synchronous message digest definition * @init: see struct ahash_alg * @update: see struct ahash_alg * @final: see struct ahash_alg * @finup: see struct ahash_alg * @digest: see struct ahash_alg * @export: see struct ahash_alg * @import: see struct ahash_alg * @setkey: see struct ahash_alg * @init_tfm: Initialize the cryptographic transformation object. * This function is called only once at the instantiation * time, right after the transformation context was * allocated. In case the cryptographic hardware has * some special requirements which need to be handled * by software, this function shall check for the precise * requirement of the transformation and put any software * fallbacks in place. * @exit_tfm: Deinitialize the cryptographic transformation object. * This is a counterpart to @init_tfm, used to remove * various changes set in @init_tfm. * @clone_tfm: Copy transform into new object, may allocate memory. * @descsize: Size of the operational state for the message digest. This state * size is the memory size that needs to be allocated for * shash_desc.__ctx * @halg: see struct hash_alg_common * @HASH_ALG_COMMON: see struct hash_alg_common */ struct shash_alg { int (*init)(struct shash_desc *desc); int (*update)(struct shash_desc *desc, const u8 *data, unsigned int len); int (*final)(struct shash_desc *desc, u8 *out); int (*finup)(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out); int (*digest)(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out); int (*export)(struct shash_desc *desc, void *out); int (*import)(struct shash_desc *desc, const void *in); int (*setkey)(struct crypto_shash *tfm, const u8 *key, unsigned int keylen); int (*init_tfm)(struct crypto_shash *tfm); void (*exit_tfm)(struct crypto_shash *tfm); int (*clone_tfm)(struct crypto_shash *dst, struct crypto_shash *src); unsigned int descsize; union { struct HASH_ALG_COMMON; struct hash_alg_common halg; }; }; #undef HASH_ALG_COMMON struct crypto_ahash { bool using_shash; /* Underlying algorithm is shash, not ahash */ unsigned int statesize; unsigned int reqsize; struct crypto_tfm base; }; struct crypto_shash { unsigned int descsize; struct crypto_tfm base; }; /** * DOC: Asynchronous Message Digest API * * The asynchronous message digest API is used with the ciphers of type * CRYPTO_ALG_TYPE_AHASH (listed as type "ahash" in /proc/crypto) * * The asynchronous cipher operation discussion provided for the * CRYPTO_ALG_TYPE_SKCIPHER API applies here as well. */ static inline struct crypto_ahash *__crypto_ahash_cast(struct crypto_tfm *tfm) { return container_of(tfm, struct crypto_ahash, base); } /** * crypto_alloc_ahash() - allocate ahash cipher handle * @alg_name: is the cra_name / name or cra_driver_name / driver name of the * ahash cipher * @type: specifies the type of the cipher * @mask: specifies the mask for the cipher * * Allocate a cipher handle for an ahash. The returned struct * crypto_ahash is the cipher handle that is required for any subsequent * API invocation for that ahash. * * Return: allocated cipher handle in case of success; IS_ERR() is true in case * of an error, PTR_ERR() returns the error code. */ struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type, u32 mask); struct crypto_ahash *crypto_clone_ahash(struct crypto_ahash *tfm); static inline struct crypto_tfm *crypto_ahash_tfm(struct crypto_ahash *tfm) { return &tfm->base; } /** * crypto_free_ahash() - zeroize and free the ahash handle * @tfm: cipher handle to be freed * * If @tfm is a NULL or error pointer, this function does nothing. */ static inline void crypto_free_ahash(struct crypto_ahash *tfm) { crypto_destroy_tfm(tfm, crypto_ahash_tfm(tfm)); } /** * crypto_has_ahash() - Search for the availability of an ahash. * @alg_name: is the cra_name / name or cra_driver_name / driver name of the * ahash * @type: specifies the type of the ahash * @mask: specifies the mask for the ahash * * Return: true when the ahash is known to the kernel crypto API; false * otherwise */ int crypto_has_ahash(const char *alg_name, u32 type, u32 mask); static inline const char *crypto_ahash_alg_name(struct crypto_ahash *tfm) { return crypto_tfm_alg_name(crypto_ahash_tfm(tfm)); } static inline const char *crypto_ahash_driver_name(struct crypto_ahash *tfm) { return crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm)); } /** * crypto_ahash_blocksize() - obtain block size for cipher * @tfm: cipher handle * * The block size for the message digest cipher referenced with the cipher * handle is returned. * * Return: block size of cipher */ static inline unsigned int crypto_ahash_blocksize(struct crypto_ahash *tfm) { return crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); } static inline struct hash_alg_common *__crypto_hash_alg_common( struct crypto_alg *alg) { return container_of(alg, struct hash_alg_common, base); } static inline struct hash_alg_common *crypto_hash_alg_common( struct crypto_ahash *tfm) { return __crypto_hash_alg_common(crypto_ahash_tfm(tfm)->__crt_alg); } /** * crypto_ahash_digestsize() - obtain message digest size * @tfm: cipher handle * * The size for the message digest created by the message digest cipher * referenced with the cipher handle is returned. * * * Return: message digest size of cipher */ static inline unsigned int crypto_ahash_digestsize(struct crypto_ahash *tfm) { return crypto_hash_alg_common(tfm)->digestsize; } /** * crypto_ahash_statesize() - obtain size of the ahash state * @tfm: cipher handle * * Return the size of the ahash state. With the crypto_ahash_export() * function, the caller can export the state into a buffer whose size is * defined with this function. * * Return: size of the ahash state */ static inline unsigned int crypto_ahash_statesize(struct crypto_ahash *tfm) { return tfm->statesize; } static inline u32 crypto_ahash_get_flags(struct crypto_ahash *tfm) { return crypto_tfm_get_flags(crypto_ahash_tfm(tfm)); } static inline void crypto_ahash_set_flags(struct crypto_ahash *tfm, u32 flags) { crypto_tfm_set_flags(crypto_ahash_tfm(tfm), flags); } static inline void crypto_ahash_clear_flags(struct crypto_ahash *tfm, u32 flags) { crypto_tfm_clear_flags(crypto_ahash_tfm(tfm), flags); } /** * crypto_ahash_reqtfm() - obtain cipher handle from request * @req: asynchronous request handle that contains the reference to the ahash * cipher handle * * Return the ahash cipher handle that is registered with the asynchronous * request handle ahash_request. * * Return: ahash cipher handle */ static inline struct crypto_ahash *crypto_ahash_reqtfm( struct ahash_request *req) { return __crypto_ahash_cast(req->base.tfm); } /** * crypto_ahash_reqsize() - obtain size of the request data structure * @tfm: cipher handle * * Return: size of the request data */ static inline unsigned int crypto_ahash_reqsize(struct crypto_ahash *tfm) { return tfm->reqsize; } static inline void *ahash_request_ctx(struct ahash_request *req) { return req->__ctx; } /** * crypto_ahash_setkey - set key for cipher handle * @tfm: cipher handle * @key: buffer holding the key * @keylen: length of the key in bytes * * The caller provided key is set for the ahash cipher. The cipher * handle must point to a keyed hash in order for this function to succeed. * * Return: 0 if the setting of the key was successful; < 0 if an error occurred */ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen); /** * crypto_ahash_finup() - update and finalize message digest * @req: reference to the ahash_request handle that holds all information * needed to perform the cipher operation * * This function is a "short-hand" for the function calls of * crypto_ahash_update and crypto_ahash_final. The parameters have the same * meaning as discussed for those separate functions. * * Return: see crypto_ahash_final() */ int crypto_ahash_finup(struct ahash_request *req); /** * crypto_ahash_final() - calculate message digest * @req: reference to the ahash_request handle that holds all information * needed to perform the cipher operation * * Finalize the message digest operation and create the message digest * based on all data added to the cipher handle. The message digest is placed * into the output buffer registered with the ahash_request handle. * * Return: * 0 if the message digest was successfully calculated; * -EINPROGRESS if data is fed into hardware (DMA) or queued for later; * -EBUSY if queue is full and request should be resubmitted later; * other < 0 if an error occurred */ int crypto_ahash_final(struct ahash_request *req); /** * crypto_ahash_digest() - calculate message digest for a buffer * @req: reference to the ahash_request handle that holds all information * needed to perform the cipher operation * * This function is a "short-hand" for the function calls of crypto_ahash_init, * crypto_ahash_update and crypto_ahash_final. The parameters have the same * meaning as discussed for those separate three functions. * * Return: see crypto_ahash_final() */ int crypto_ahash_digest(struct ahash_request *req); /** * crypto_ahash_export() - extract current message digest state * @req: reference to the ahash_request handle whose state is exported * @out: output buffer of sufficient size that can hold the hash state * * This function exports the hash state of the ahash_request handle into the * caller-allocated output buffer out which must have sufficient size (e.g. by * calling crypto_ahash_statesize()). * * Return: 0 if the export was successful; < 0 if an error occurred */ int crypto_ahash_export(struct ahash_request *req, void *out); /** * crypto_ahash_import() - import message digest state * @req: reference to ahash_request handle the state is imported into * @in: buffer holding the state * * This function imports the hash state into the ahash_request handle from the * input buffer. That buffer should have been generated with the * crypto_ahash_export function. * * Return: 0 if the import was successful; < 0 if an error occurred */ int crypto_ahash_import(struct ahash_request *req, const void *in); /** * crypto_ahash_init() - (re)initialize message digest handle * @req: ahash_request handle that already is initialized with all necessary * data using the ahash_request_* API functions * * The call (re-)initializes the message digest referenced by the ahash_request * handle. Any potentially existing state created by previous operations is * discarded. * * Return: see crypto_ahash_final() */ int crypto_ahash_init(struct ahash_request *req); /** * crypto_ahash_update() - add data to message digest for processing * @req: ahash_request handle that was previously initialized with the * crypto_ahash_init call. * * Updates the message digest state of the &ahash_request handle. The input data * is pointed to by the scatter/gather list registered in the &ahash_request * handle * * Return: see crypto_ahash_final() */ int crypto_ahash_update(struct ahash_request *req); /** * DOC: Asynchronous Hash Request Handle * * The &ahash_request data structure contains all pointers to data * required for the asynchronous cipher operation. This includes the cipher * handle (which can be used by multiple &ahash_request instances), pointer * to plaintext and the message digest output buffer, asynchronous callback * function, etc. It acts as a handle to the ahash_request_* API calls in a * similar way as ahash handle to the crypto_ahash_* API calls. */ /** * ahash_request_set_tfm() - update cipher handle reference in request * @req: request handle to be modified * @tfm: cipher handle that shall be added to the request handle * * Allow the caller to replace the existing ahash handle in the request * data structure with a different one. */ static inline void ahash_request_set_tfm(struct ahash_request *req, struct crypto_ahash *tfm) { req->base.tfm = crypto_ahash_tfm(tfm); } /** * ahash_request_alloc() - allocate request data structure * @tfm: cipher handle to be registered with the request * @gfp: memory allocation flag that is handed to kmalloc by the API call. * * Allocate the request data structure that must be used with the ahash * message digest API calls. During * the allocation, the provided ahash handle * is registered in the request data structure. * * Return: allocated request handle in case of success, or NULL if out of memory */ static inline struct ahash_request *ahash_request_alloc_noprof( struct crypto_ahash *tfm, gfp_t gfp) { struct ahash_request *req; req = kmalloc_noprof(sizeof(struct ahash_request) + crypto_ahash_reqsize(tfm), gfp); if (likely(req)) ahash_request_set_tfm(req, tfm); return req; } #define ahash_request_alloc(...) alloc_hooks(ahash_request_alloc_noprof(__VA_ARGS__)) /** * ahash_request_free() - zeroize and free the request data structure * @req: request data structure cipher handle to be freed */ static inline void ahash_request_free(struct ahash_request *req) { kfree_sensitive(req); } static inline void ahash_request_zero(struct ahash_request *req) { memzero_explicit(req, sizeof(*req) + crypto_ahash_reqsize(crypto_ahash_reqtfm(req))); } static inline struct ahash_request *ahash_request_cast( struct crypto_async_request *req) { return container_of(req, struct ahash_request, base); } /** * ahash_request_set_callback() - set asynchronous callback function * @req: request handle * @flags: specify zero or an ORing of the flags * CRYPTO_TFM_REQ_MAY_BACKLOG the request queue may back log and * increase the wait queue beyond the initial maximum size; * CRYPTO_TFM_REQ_MAY_SLEEP the request processing may sleep * @compl: callback function pointer to be registered with the request handle * @data: The data pointer refers to memory that is not used by the kernel * crypto API, but provided to the callback function for it to use. Here, * the caller can provide a reference to memory the callback function can * operate on. As the callback function is invoked asynchronously to the * related functionality, it may need to access data structures of the * related functionality which can be referenced using this pointer. The * callback function can access the memory via the "data" field in the * &crypto_async_request data structure provided to the callback function. * * This function allows setting the callback function that is triggered once * the cipher operation completes. * * The callback function is registered with the &ahash_request handle and * must comply with the following template:: * * void callback_function(struct crypto_async_request *req, int error) */ static inline void ahash_request_set_callback(struct ahash_request *req, u32 flags, crypto_completion_t compl, void *data) { req->base.complete = compl; req->base.data = data; req->base.flags = flags; } /** * ahash_request_set_crypt() - set data buffers * @req: ahash_request handle to be updated * @src: source scatter/gather list * @result: buffer that is filled with the message digest -- the caller must * ensure that the buffer has sufficient space by, for example, calling * crypto_ahash_digestsize() * @nbytes: number of bytes to process from the source scatter/gather list * * By using this call, the caller references the source scatter/gather list. * The source scatt |